diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..e18365b30 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "images/proxy/nginx-lua-prometheus"] + path = images/proxy/nginx-lua-prometheus + url = https://github.com/knyar/nginx-lua-prometheus.git diff --git a/ansible/api-manager.yml b/ansible/api-manager.yml index 8413d2495..ef78b4bb4 100644 --- a/ansible/api-manager.yml +++ b/ansible/api-manager.yml @@ -1,6 +1,8 @@ --- - name: Kong API become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" hosts: "{{groups['swarm-manager-1'][0]}}" roles: - {role: kong-api, tags: ['kong-api']} diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml index a58a6f62d..feb78219b 100644 --- a/ansible/artifacts-download.yml +++ b/ansible/artifacts-download.yml @@ -4,7 +4,7 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ artifact_azure_account_name }}" - AZURE_STORAGE_KEY: "{{ artifact_azure_account_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_artifact_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_artifact_storage_account_sas }}" roles: - artifacts-download-azure diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml index 4071e5132..772ec2cca 100644 --- a/ansible/artifacts-upload.yml +++ b/ansible/artifacts-upload.yml @@ -4,7 +4,7 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ artifact_azure_account_name }}" - AZURE_STORAGE_KEY: "{{ artifact_azure_account_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_artifact_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_artifact_storage_account_sas }}" roles: - artifacts-upload-azure diff --git a/ansible/artifacts/sunbird/login/info.ftl b/ansible/artifacts/sunbird/login/info.ftl index f151cfde5..d52d7af69 100755 --- a/ansible/artifacts/sunbird/login/info.ftl +++ b/ansible/artifacts/sunbird/login/info.ftl @@ -3,23 +3,17 @@ <#if section = "title"> ${message.summary} <#elseif section = "form"> -
-
-
-
+
+
+

${message.summary}

<#if skipLink??> <#else> - <#if pageRedirectUri??> -

${msg("backToApplication")}

- <#elseif client.baseUrl??> -

${msg("backToApplication")}

- +

${msg("backToApplication")}

-
diff --git a/ansible/artifacts/sunbird/login/login-page-expired.ftl b/ansible/artifacts/sunbird/login/login-page-expired.ftl index b26661442..9bed0120b 100644 --- a/ansible/artifacts/sunbird/login/login-page-expired.ftl +++ b/ansible/artifacts/sunbird/login/login-page-expired.ftl @@ -3,16 +3,16 @@ <#if section = "title"> ${msg("pageExpiredTitle")} <#elseif section = "form"> -
-
-
-
-

- ${msg("pageExpiredMsg1")} ${msg("doClickHere")}. - ${msg("pageExpiredMsg2")} ${msg("doClickHere")}. -

+
+
+
+

${msg("pageExpiredMsg1")}
+ ${msg("doClickHere")}

+

+ ${msg("pageExpiredMsg2")}
+ ${msg("doClickHere")} +

-
diff --git a/ansible/artifacts/sunbird/login/login-reset-password.ftl b/ansible/artifacts/sunbird/login/login-reset-password.ftl index 5d81b0c7a..839c4c0a1 100644 --- a/ansible/artifacts/sunbird/login/login-reset-password.ftl +++ b/ansible/artifacts/sunbird/login/login-reset-password.ftl @@ -4,18 +4,16 @@ ${msg("emailForgotTitle")} <#elseif section = "form"> -
-
-
-
-
+
+
+
-
${msg("emailForgotTitle")}
+
${msg("emailForgotTitle")}
-
+
${msg("enterEmailPhonenumberToGetCode")}
-
+
<#if message?has_content>
${message.summary} @@ -23,7 +21,7 @@
-
+
-
-
- << ${msg("backToLogin")} +
+ ${msg("backToLogin")}
-
-
<#elseif section = "info" > diff --git a/ansible/artifacts/sunbird/login/login-update-password.ftl b/ansible/artifacts/sunbird/login/login-update-password.ftl index 434a26985..6bb0167c0 100644 --- a/ansible/artifacts/sunbird/login/login-update-password.ftl +++ b/ansible/artifacts/sunbird/login/login-update-password.ftl @@ -3,15 +3,13 @@ <#if section = "title"> ${msg("updatePasswordTitle")} <#elseif section = "form"> -
-
-
-
+
+
${msg("newPasswordTitle")}
-
+
<#if message?has_content>
${message.summary} @@ -28,10 +26,11 @@
- +
+
Your password must contain a minimum of 8 characters. It must include numerals, lower and upper case alphabets and special characters, without any spaces
@@ -41,10 +40,11 @@ - + +
Passwords do not match
-
@@ -56,8 +56,6 @@ -
-
diff --git a/ansible/artifacts/sunbird/login/login.ftl b/ansible/artifacts/sunbird/login/login.ftl index fdf5abbbe..a7824bbf1 100644 --- a/ansible/artifacts/sunbird/login/login.ftl +++ b/ansible/artifacts/sunbird/login/login.ftl @@ -5,17 +5,16 @@ <#elseif section = "header"> <#elseif section = "form"> <#if realm.password> -
-
-
-
-
+
+
+
-
${msg("loginDiksha")}
+
${msg("loginDiksha")}
+

Login

${msg("mergeAccountMessage")}

${msg("migrateAccountMessage")}

-
+
<#if message?has_content>
${message.summary} @@ -24,7 +23,7 @@
suceess
error
-
+
- <#if usernameEditDisabled??> - <#-- TODO: need to find alternative for prepopulating username --> - + <#if usernameEditDisabled??> + <#-- TODO: need to find alternative for prepopulating username --> + <#else> - + +
-
+
- <#if realm.resetPasswordAllowed> - ${msg("doForgotPassword")} - - +
- + ${msg("inCorrectPasswordError")}
-
- +
+
+ + +
+
+ <#if realm.resetPasswordAllowed> + ${msg("doForgotPassword")} + + +
+
+ +
+ <#if realm.password && realm.registrationAllowed && !usernameEditDisabled??> +
+
+ ${msg("noAccount")} ${msg("registerHere")} +
+
+

OR

@@ -76,28 +92,22 @@ - - +
- <#if realm.password && realm.registrationAllowed && !usernameEditDisabled??> -
-
- ${msg("noAccount")} ${msg("registerHere")} -
-
-
- ${msg("goBack")} -
-
-
+ ${msg("goBack")} + +
- + \ No newline at end of file diff --git a/ansible/artifacts/sunbird/login/messages/messages_en.properties b/ansible/artifacts/sunbird/login/messages/messages_en.properties index b9d375c91..dc7b00b0f 100644 --- a/ansible/artifacts/sunbird/login/messages/messages_en.properties +++ b/ansible/artifacts/sunbird/login/messages/messages_en.properties @@ -1,6 +1,8 @@ doLogIn=Login +login=LOGIN doRegister=Sign Up doSignIn=Sign In +signIn=Sign in doSignWithGoogle=with Google doSignWithState=Login with State System doCancel=Cancel @@ -11,7 +13,7 @@ doNo=No doContinue=Continue doAccept=Accept doDecline=Decline -doForgotPassword=Forgot? +doForgotPassword=Forget password? doClickHere=Click here doImpersonate=Impersonate kerberosNotConfigured=Kerberos Not Configured @@ -53,8 +55,8 @@ consentDenied=Consent denied. noAccount=Don''t have an account? username=Username or Mobile number goBack=<<Go Back -mergeAccountMessage=Enter Mobile number / Email Address OR use Google Sign in to identify the account from which you want to merge usage details -migrateAccountMessage=Confirm the password for the DIKSHA account you want to merge of cick Sign in with Google to sign in using your Gmail account +mergeAccountMessage=Enter Mobile number / Email Address OR use Login with Google to identify the account from which you want to merge usage details +migrateAccountMessage=Confirm the password for the DIKSHA account you want to merge of click Login with Google to sign in using your Gmail account inCorrectPasswordError=The password entered is incorrect. Enter the password again. emailOrPhone=Email Address / Mobile number placeholderForEmailOrPhone=Enter Email Address / Mobile number @@ -268,6 +270,6 @@ requiredAction.VERIFY_EMAIL=Verify Email user_not_found=This Email Address/Mobile Number doesn''t belong to a valid user p3pPolicy=CP="This is not a P3P policy!" usernamePlaceholder = Enter your Registered Email address/Mobile number -passwordPlaceholder = Enter password -loginDiksha = Log into DIKSHA +passwordPlaceholder = Enter your password +loginDiksha = Welcome to SUNBIRD registerHere = Register here diff --git a/ansible/artifacts/sunbird/login/resources/css/login.css b/ansible/artifacts/sunbird/login/resources/css/login.css index f9255e2cf..b84a6a8ee 100644 --- a/ansible/artifacts/sunbird/login/resources/css/login.css +++ b/ansible/artifacts/sunbird/login/resources/css/login.css @@ -3,13 +3,18 @@ body{ font-family: "Noto Sans"; position: relative; letter-spacing: 0.2px; + background: #edf4f9; } -.ui.container.fullpage-background-image{ - max-width: 320px !important; +.fullpage-background-image{ + width: 100% !important; margin: 0 auto !important; box-shadow: none !important; border: 0 !important; border-radius: 0 !important; + background: #EDF4F9; + display: flex; + align-items: center; + min-height: auto; } .fs-14{ font-size: 14px !important; @@ -17,24 +22,21 @@ body{ .fs-22{ font-size: 22px !important; } -@media screen and (min-width: 768px) { - body{ - background-size: cover; - background-repeat: no-repeat; - min-height: 100vh; - background: url(../img/bg.svg) no-repeat center center fixed; - background-size: cover; - - } - .ui.container.fullpage-background-image{ - max-width: 944px !important; - padding: 20px 0 !important; - margin: 30px auto!important; - box-shadow: 0 2px 16px 0 rgba(0,0,0,0.2) !important; - border-radius: 4px !important; - border: 1px solid #d8dee2 !important; - } +.passwdchk { + margin-top: 10px; + font-size: 10.0002px!important; + color: #999999; +} +.passwderr { + margin-top: 10px; + font-size: 10.0002px!important; + color: #ff6979 !important; +} +.confpasswderr { + margin-top: 10px; + color: #ff6979 !important; } + .ui.button:focus{ box-shadow: 0 2px 8px 0 rgba(0,0,0,.16) !important; } @@ -54,10 +56,10 @@ input:focus{ border: 1px solid #024f9d !important; } .signInWithGoogle{ - /* vertical-align: bottom; */ - height: 18px !important; - margin-right: 8px; - + position: absolute; + left: 1px; + top: 1px; + width: 34px; } .ui.basic.blue.button.googleButton{ color: #333 !important; @@ -87,11 +89,12 @@ input:focus{ background-repeat:no-repeat; font-weight: 700; font-size: 12px; + opacity: 50%; } .signUpMsg{ font-size: 12px; line-height: 17px; - text-align: center; + text-align: left; /*margin-top: auto !important;*/ margin-bottom: 10px !important; } @@ -100,11 +103,23 @@ input:focus{ line-height: 22px; } .signInHead{ - font-size: 14px; + color: #024F9D; + font-size: 24px; + font-weight: bold; + letter-spacing: 0; + line-height: 32px; + text-align: center; +} +p.subtitle { + color: #333333; + font-size: 18px; font-weight: bold; + letter-spacing: 0; + line-height: 26px; + text-align: center; } .ui.form .field>label { - font-size: 1.05em; + font-size: 14px; font-weight: 500; } .forgetPasswordLink{ @@ -123,12 +138,33 @@ input:focus{ .textCenter { text-align: center !important; } +.mb-16{ + margin-bottom: 16px !important; +} +.mb-24{ + margin-bottom: 24px !important; +} .mb-28{ margin-bottom: 28px !important; } .mb-0{ margin-bottom: 0px !important; } +.mb-4 { + margin-bottom: 4px !important; +} +.ml-4 { + margin-left: 4px !important; +} +.mb-8 { + margin-bottom: 8px !important; +} +.mt-4 { + margin-top: 4px !important; +} +.mt-8 { + margin-top: 8px !important; +} .mt-46{ margin-top: 46px; } @@ -141,6 +177,9 @@ input:focus{ .mt-16 { margin-top: 16px !important; } +.mt-24 { + margin-top: 24px !important; +} .mb-18 { margin-bottom: 18px !important; } @@ -185,7 +224,6 @@ input:focus{ max-width: 100% !important; width: auto !important; max-height: 56px !important; - margin-top: 20px!important; } .button{ color: #F9F9F9 !important; @@ -195,16 +233,19 @@ input:focus{ } .ui.text.success{ - color: #33B17B; + color: #008840; } .ui.text.warning{ - color: #794b02; + color: #e55a28; } .ui.text.error{ - color:#9f3a38; + color:#ff4558; + font-size: 14px; + display: block; + margin-top: 8px;margin-bottom: 8px; } .ui.text.info{ - color:#276f86; + color:#024f9d; } .hide{ display: none !important; @@ -229,6 +270,10 @@ a{ outline:none; cursor: pointer; } + +.pointer { + cursor: pointer; +} .ui.blue.button{ box-shadow: 0 0 0 1px #024f9d inset!important; } @@ -312,14 +357,14 @@ h1, h2, h3, h4, h5, outline: 0; display: inline-block; border: 1px solid; - color: var(--white); + color: #fff; cursor: pointer; min-width: 64px; border-radius: 3px; -webkit-border-radius: 3px; -moz-border-radius: 3px; position: relative; - text-transform: capitalize; + text-transform: inherit; text-decoration: none; } @@ -388,4 +433,193 @@ h1, h2, h3, h4, h5, .flex-jc-center { justify-content: center !important; +} + +.instruction p { + font-size:14px; +} +.line-height-normal { + line-height: normal; +} +/* new layout */ +#kc-form-wrapper { + display: flex !important; + align-items: center; + justify-content: center; + height: 100%; + padding: 32px; + background: #edf4f9; +} +.container-wrapper { + max-width: 328px; + width:100%; + margin:0 auto; + min-height: auto; + display: flex; + flex-direction: column; + justify-content: center; +} +.remember-forgot-row { + display: flex; + flex-direction: row-reverse; + justify-content: space-between; + align-items: center; +} +.forgot-password { + font-size:11px; + color:#333; +} +.forgot-password:hover { + color:#024f9d; + text-decoration: underline; + cursor: pointer; +} +.sb-checkbox, .sb-radio-btn-checkbox { + position: relative; + display: block; + -webkit-backface-visibility: hidden; + backface-visibility: hidden; + outline: 0; + font-style: normal; + line-height: 1rem; + min-height: 1rem; + min-width: 1rem; +} +.sb-checkbox input[type="checkbox"] { + outline: 0; + opacity: 0; +} +.sb-checkbox input[type="radio"], .sb-checkbox input[type="checkbox"], .sb-radio-btn-checkbox input[type="radio"], .sb-radio-btn-checkbox input[type="checkbox"] { + cursor: pointer; + position: absolute; + top: 0; + left: 0; + opacity: 0; + z-index: 3; + width: 1rem; + height: 1rem; +} +[type=checkbox], [type=radio] { + box-sizing: border-box; + padding: 0; +} +.sb-checkbox label, .sb-radio-btn-checkbox label { + cursor: pointer; + position: relative; + display: block; + padding-left: 1.5rem; + outline: 0; + font-size: 0.8571rem; + font-weight: normal; + margin-right: 0.5rem; +} +.sb-checkbox.sb-checkbox-secondary input:checked ~ label:before { + background-color: #00c786; + border-color: #00c786; + border-radius: 50%; +} +.sb-checkbox input:checked ~ label:before { + background-color: #024f9d; + border-color: #024f9d; +} +.sb-checkbox label:before { + border-radius: 0.25rem; +} +.sb-checkbox label:before, .sb-radio-btn-checkbox label:before { + content: "\2713"; + position: absolute; + top: 0; + left: 0; + width: 1rem; + height: 1rem; + background: #fff; + transition: border 0.1s ease, opacity 0.1s ease, transform 0.1s ease, box-shadow 0.1s ease; + border: 0.0625rem solid #999; + border-radius: 50%; + color: #999; + padding-left:1px; +} +.sb-checkbox input:checked ~ .box:after, .sb-checkbox input:checked ~ label:after { + content: "\2713"; + border-radius: 0.25rem; + font-weight: normal; + color: #fff !important; +} + +.sb-checkbox input:checked ~ .box:after, .sb-checkbox input:checked ~ label:after { + opacity: 1; +} +.sb-checkbox label:after { + position: absolute; + font-size: 0.8571rem; + top: 0; + left: 0; + width: 1rem; + height: 1rem; + text-align: center; + opacity: 0; + color: #fff; + transition: border 0.1s ease, opacity 0.1s ease, transform 0.1s ease, box-shadow 0.1s ease; +} + +.sb-btn-outline-gray { + background-color: #fff; + color: #024f9d; + border-color:#999; +} +.back-btn { + position: absolute; + left: 8px; + top: 16px; + min-width: auto; + display: flex; + justify-content: center; + align-items: center; +} +.pull-left { + float:left; +} +.pull-right { + float:right; +} +.sb-btn-link { + background: none; + border: 0; +} +.sb-btn-link-primary { + background: none; + border: 0px; + color: #024f9d; +} + +@media screen and (min-width: 768px) { + body{ + background-size: cover; + background-repeat: no-repeat; + min-height: 100vh; + background: url(../img/onboard_bg.svg) no-repeat center center fixed; + background-size: cover; + + } + .fullpage-background-image{ + max-width: 1008px !important; + width:100%; + padding: 40px !important; + /* box-shadow: 0 2px 16px 0 rgba(0,0,0,0.2) !important; */ + border-radius: 8px !important; + min-height: 600px; + } + #kc-form-wrapper { + height: 100vh; + padding: 32px; + background: none; + } + .container-wrapper { + min-height: auto; + } +} + +.btn-signInWithGoogle { + background: #4285F4; + border: #4285F4; } \ No newline at end of file diff --git a/ansible/artifacts/sunbird/login/resources/img/arrow_back.png b/ansible/artifacts/sunbird/login/resources/img/arrow_back.png new file mode 100644 index 000000000..87cca9e86 Binary files /dev/null and b/ansible/artifacts/sunbird/login/resources/img/arrow_back.png differ diff --git a/ansible/artifacts/sunbird/login/resources/img/google.svg b/ansible/artifacts/sunbird/login/resources/img/google.svg new file mode 100644 index 000000000..cc0184eb0 --- /dev/null +++ b/ansible/artifacts/sunbird/login/resources/img/google.svg @@ -0,0 +1,21 @@ + + + + E9824911-08AE-4CF9-8FC4-7A3718DAA2D2 + Created with sketchtool. + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ansible/artifacts/sunbird/login/resources/img/onboard_bg.svg b/ansible/artifacts/sunbird/login/resources/img/onboard_bg.svg new file mode 100644 index 000000000..c726973a7 --- /dev/null +++ b/ansible/artifacts/sunbird/login/resources/img/onboard_bg.svg @@ -0,0 +1,25 @@ + + + + C92B0B70-3C7D-4A4B-9DB8-F758FF86CD1B@3x + Created with sketchtool. + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ansible/artifacts/sunbird/login/resources/js/login.js b/ansible/artifacts/sunbird/login/resources/js/login.js index 8c4b5e5e0..06cfb9011 100644 --- a/ansible/artifacts/sunbird/login/resources/js/login.js +++ b/ansible/artifacts/sunbird/login/resources/js/login.js @@ -88,11 +88,66 @@ window.onload = function(){ } var autoMerge = getValueFromSession('automerge'); if (autoMerge === '1') { + var isValuePresent = (new URLSearchParams(window.location.search)).get('automerge'); + if (isValuePresent) { + sessionStorage.removeItem('session_url'); + initialize(); + } decoratePage('autoMerge'); storeValueForMigration(); } }; +var validatePassword = function () { + setTimeout(() => { + var textInput = document.getElementById("password-new").value; + var text2Input = document.getElementById("password-confirm").value; + var charRegex = new RegExp("^(?=.{8,})"); + var lwcsRegex = new RegExp("^(?=.*[a-z])"); + var spaceRegex = new RegExp('^\\S*$'); + var upcsRegex = new RegExp("^(?=.*[A-Z])"); + var numRegex = new RegExp("^(?=.*[0-9])"); + var specRegex = new RegExp('^[!"#$%&\'()*+,-./:;<=>?@[^_`{|}~\]]'); + var error_msg = document.getElementById('passwd-error-msg'); + var match_error_msg = document.getElementById('passwd-match-error-msg'); + if (charRegex.test(textInput) && spaceRegex.test(textInput) && lwcsRegex.test(textInput) && upcsRegex.test(textInput) && numRegex.test(textInput) && !specRegex.test(textInput)) { + error_msg.className = error_msg.className.replace("passwderr","passwdchk"); + if (textInput === text2Input) { + match_error_msg.className = match_error_msg.className.replace("show","hide"); + document.getElementById("login").disabled = false; + } + } else { + error_msg.className = error_msg.className.replace("passwdchk","passwderr"); + } + if (textInput !== text2Input) { + document.getElementById("login").disabled = true; + } + }); +} + +var matchPassword = function () { + setTimeout(() => { + var textInput = document.getElementById("password-new").value; + var text2Input = document.getElementById("password-confirm").value; + var charRegex = new RegExp("^(?=.{8,})"); + var spaceRegex = new RegExp('^\\S*$'); + var lwcsRegex = new RegExp("^(?=.*[a-z])"); + var upcsRegex = new RegExp("^(?=.*[A-Z])"); + var numRegex = new RegExp("^(?=.*[0-9])"); + var specRegex = new RegExp('^[!"#$%&\'()*+,-./:;<=>?@[^_`{|}~\]]'); + var match_error_msg = document.getElementById('passwd-match-error-msg'); + if (textInput === text2Input) { + if (charRegex.test(text2Input) && spaceRegex.test(textInput) && lwcsRegex.test(text2Input) && upcsRegex.test(text2Input) && numRegex.test(text2Input) && !specRegex.test(text2Input)) { + match_error_msg.className = match_error_msg.className.replace("show","hide"); + document.getElementById("login").disabled = false; + } + } else { + match_error_msg.className = match_error_msg.className.replace("hide","show"); + document.getElementById("login").disabled = true; + } + }); +} + var storeValueForMigration = function () { // storing values in sessionStorage for future references sessionStorage.setItem('automerge', getValueFromSession('automerge')); @@ -100,6 +155,8 @@ var storeValueForMigration = function () { sessionStorage.setItem('identifierValue', getValueFromSession('identifierValue')); sessionStorage.setItem('identifierType', getValueFromSession('identifierType')); sessionStorage.setItem('userId', getValueFromSession('userId')); + sessionStorage.setItem('tncAccepted', getValueFromSession('tncAccepted')); + sessionStorage.setItem('tncVersion', getValueFromSession('tncVersion')); }; var getValueFromSession = function (valueId) { var value = (new URLSearchParams(window.location.search)).get(valueId); @@ -180,7 +237,8 @@ var handlePasswordFailure = function () { sessionStorage.setItem('passwordFailCount', passwordFailCount); if (passwordFailCount >= 2) { const url = '/sign-in/sso/auth?status=error' + '&identifierType=' + getValueFromSession('identifierType'); - const query = '&userId=' + getValueFromSession('userId') + '&identifierValue=' + getValueFromSession('identifierValue'); + let query = '&userId=' + getValueFromSession('userId') + '&identifierValue=' + getValueFromSession('identifierValue'); + query = query + '&tncAccepted=' + getValueFromSession('tncAccepted') + '&tncVersion=' + getValueFromSession('tncVersion'); window.location.href = window.location.protocol + '//' + window.location.host + url + query; } }; @@ -337,6 +395,14 @@ var forgetPassword = (redirectUrlPath) => { } } +var backToApplication = () => { + var redirect_uri = getValueFromSession('redirect_uri'); + if (redirect_uri) { + var updatedQuery = redirect_uri.split('?')[0]; + window.location.href = updatedQuery; + } +} + var redirect = (redirectUrlPath) => { console.log('redirect', redirectUrlPath) const curUrlObj = window.location; @@ -397,7 +463,11 @@ var handleGoogleAuthEvent = () => { if (redirect_uri) { const redirect_uriLocation = new URL(redirect_uri); if (client_id === 'android') { - const googleRedirectUrl = sessionUrlObj.protocol + '//' + sessionUrlObj.host + googleAuthUrl; + let host = sessionUrlObj.host; + if (host.indexOf("merge.") !== -1) { + host = host.slice(host.indexOf("merge.") + 6, host.length); + } + const googleRedirectUrl = sessionUrlObj.protocol + '//' + host + googleAuthUrl; window.location.href = redirect_uri + '?googleRedirectUrl=' + googleRedirectUrl + updatedQuery; } else { window.location.href = redirect_uriLocation.protocol + '//' + redirect_uriLocation.host + googleAuthUrl + updatedQuery; diff --git a/ansible/artifacts/sunbird/login/template.ftl b/ansible/artifacts/sunbird/login/template.ftl index 23e7579f5..ac560711e 100644 --- a/ansible/artifacts/sunbird/login/template.ftl +++ b/ansible/artifacts/sunbird/login/template.ftl @@ -86,7 +86,7 @@ if(sessionTenant){ var imgSrc = "${url.resourcesPath}/img/tenants/"+sessionTenant+".png"; }else{ - var imgSrc = "${url.resourcesPath}/img/diksha_gov_logo.svg"; + var imgSrc = "${url.resourcesPath}/img/logo.png"; } var logoImg = document.querySelector(".ui.header img"); @@ -94,7 +94,7 @@ logoImg.setAttribute('class','logo-image'); logoImg.setAttribute('alt',sessionTenant); logoImg.src = imgSrc; - logoImg.addEventListener("error", ()=>{ logoImg.onerror=null;logoImg.src='${url.resourcesPath}/img/diksha_gov_logo.svg'}); + logoImg.addEventListener("error", ()=>{ logoImg.onerror=null;logoImg.src='${url.resourcesPath}/img/logo.png'}); } diff --git a/ansible/cassandra-backup.yml b/ansible/cassandra-backup.yml index 53662af23..fdfa7933d 100644 --- a/ansible/cassandra-backup.yml +++ b/ansible/cassandra-backup.yml @@ -1,6 +1,23 @@ +# This task will take snapshot serially for all cassandra nodes +# If we club both taking snapshot and upload the data, then the +# interval of snapshots b/w nodes will be very high; Which might lead +# to data discrepancies +- hosts: cassandra + become: yes + serial: true + tasks: + - name: taking cassandra snapshots + shell: | + nodetool clearsnapshot + nodetool snapshot -t "cassandra-backup-{{ lookup('pipe', 'date +%Y%m%d') }}-{{ ansible_hostname }}-new" + +# Once snaphot is done, +# We can take the snapshot and compress it and upload it +# This will take some cpu and memory in the nodes +# Because of that we're running it serially, so that it won't affect the perfomance - hosts: cassandra become: yes vars_files: - ['{{ inventory_dir }}/secrets.yml'] roles: - - cassandra-backup + - {name: cassandra-backup, vars: [ additional_arguments: "--disablesnapshot"]} diff --git a/ansible/cassandra-db-update.yml b/ansible/cassandra-db-update.yml new file mode 100644 index 000000000..008123619 --- /dev/null +++ b/ansible/cassandra-db-update.yml @@ -0,0 +1,7 @@ +- hosts: cassandra + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - cassandra-db-update + run_once: true diff --git a/ansible/cassandra-deploy-decrypt.yml b/ansible/cassandra-deploy-decrypt.yml index 341631a4b..fa0053cdc 100644 --- a/ansible/cassandra-deploy-decrypt.yml +++ b/ansible/cassandra-deploy-decrypt.yml @@ -5,6 +5,7 @@ sunbird_cassandra_port: 9042 sunbird_cassandra_keyspace: sunbird sunbird_encryption_key: "{{ core_vault_sunbird_encryption_key }}" + sunbird_cassandra_groups_keyspace: sunbird_groups vars_files: - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] roles: diff --git a/ansible/cassandra-exporter.yaml b/ansible/cassandra-exporter.yaml new file mode 100644 index 000000000..a5802cab7 --- /dev/null +++ b/ansible/cassandra-exporter.yaml @@ -0,0 +1,6 @@ +- hosts: cassandra + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - cassandra-exporter diff --git a/ansible/cert-file-upload.yml b/ansible/cert-file-upload.yml new file mode 100644 index 000000000..e29a7b6c2 --- /dev/null +++ b/ansible/cert-file-upload.yml @@ -0,0 +1,10 @@ +--- +- hosts: localhost + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_public_storage_account_key }}" + roles: + - cert-file-upload diff --git a/ansible/deploy-plugins.yml b/ansible/deploy-plugins.yml index 648c95c09..cd2b5b512 100644 --- a/ansible/deploy-plugins.yml +++ b/ansible/deploy-plugins.yml @@ -4,8 +4,8 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ azure_plugin_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ azure_plugin_storage_account_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_public_storage_account_sas }}" tasks: - name: rename env_domain in preview_cdn.html for CDN shell: | @@ -17,14 +17,11 @@ - name: delete batch shell: | - sas_expire_time=`date +'%Y-%m-%dT%H:%m:%SZ' -d '-4 hours'` - echo $sas_expire_time - sas_token=`az storage account generate-sas --account-name "{{ azure_plugin_storage_account_name }}" --account-key "{{ azure_plugin_storage_account_key }}" --expiry $sas_expire_time --https-only --permissions ld --resource-types sco --services bfqt | sed -E 's/^"|"$//g'` - azcopy rm "https://"{{ azure_plugin_storage_account_name }}".blob.core.windows.net/"{{ plugin_container_name }}"/"{{ folder_name }}"?$sas_token" --recursive=true + azcopy rm "https://{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ plugin_container_name }}/{{ folder_name }}{{sunbird_public_storage_account_sas}}" --recursive=true async: 3600 poll: 10 tags: - - content-editor + - content-editor - collection-editor - generic-editor - preview diff --git a/ansible/deploy.yml b/ansible/deploy.yml index 1cc400f74..94d72a823 100644 --- a/ansible/deploy.yml +++ b/ansible/deploy.yml @@ -8,6 +8,14 @@ - stack-proxy run_once: true +- hosts: swarm-bootstrap-manager + become: yes + roles: + - stack-proxy-private + tags: + - stack-proxy-private + run_once: true + - hosts: swarm-bootstrap-manager become: yes vars_files: diff --git a/ansible/desktop-faq-upload.yml b/ansible/desktop-faq-upload.yml new file mode 100644 index 000000000..ae2abc086 --- /dev/null +++ b/ansible/desktop-faq-upload.yml @@ -0,0 +1,32 @@ +- hosts: localhost + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + vars: + artifacts_container: "{{desktop_container}}" + artifact: "{{destination_path}}" + artifact_path: "{{playbook_dir}}/../{{src_file_path}}" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_public_storage_account_key }}" + roles: + - artifacts-upload-azure + tags: + - upload-desktop-faq + + +- hosts: localhost + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + vars: + artifacts_container: "{{desktop_container}}" + artifact: "{{destination_path}}" + artifact_path: "{{playbook_dir}}/../{{src_file_path}}" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_private_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_private_storage_account_key }}" + roles: + - artifacts-upload-azure + tags: + - upload-label diff --git a/ansible/docker_image_push.yml b/ansible/docker_image_push.yml index 5b0dc91f0..6a5eefa34 100644 --- a/ansible/docker_image_push.yml +++ b/ansible/docker_image_push.yml @@ -1,6 +1,7 @@ --- - name: "Push docker images to any docker registry" - hosts: localhost + hosts: local + become: yes vars_files: - ['{{inventory_dir}}/secrets.yml'] become: yes @@ -10,6 +11,7 @@ registry: "{{ vault_docker_registry_url }}" username: "{{ vault_docker_registry_user }}" password: "{{ vault_docker_registry_password }}" + tags: docker-login - name: Push image to registry docker_image: diff --git a/ansible/es-restore.yml b/ansible/es-restore.yml deleted file mode 100644 index 2b2bb48d1..000000000 --- a/ansible/es-restore.yml +++ /dev/null @@ -1,25 +0,0 @@ -- hosts: "{{remote}}" - gather_facts: no - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - vars: - - es_restore_host: "{{remote}}" - - snapshot_base_path: "{{app_snapshot_base_path}}" - roles: - - es-azure-restore - tags: - - es_restore - -- hosts: "{{remote}}" - gather_facts: no - become: yes - vars_files: - - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] - vars: - - es_restore_host: "{{remote}}" - - snapshot_base_path: "{{log_snapshot_base_path}}" - roles: - - es-azure-restore - tags: - - log_es_restore diff --git a/ansible/es_restore.yml b/ansible/es_restore.yml new file mode 100644 index 000000000..ed91cb1e4 --- /dev/null +++ b/ansible/es_restore.yml @@ -0,0 +1,9 @@ +--- +- hosts: "{{ es_restore_host }}" + become: yes + vars_files: + - ['{{inventory_dir}}/secrets.yml'] + roles: + - es-azure-restore + tags: + - es_restore diff --git a/ansible/esupgradelatest.yml b/ansible/esupgradelatest.yml index ce5fffafd..d14ea66e0 100644 --- a/ansible/esupgradelatest.yml +++ b/ansible/esupgradelatest.yml @@ -1,24 +1,53 @@ -- hosts: log-es-2 +- hosts: log-es become: yes vars_files: - ['{{inventory_dir}}/secrets.yml'] - tasks: + pre_tasks: - name: Registering node name set_fact: es_instance_name: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}es-{{ loop.index }}{% endif %}{% endfor %}" roles: - openjdk - - { role: es6, + - { role: log-es6, es_config: { - cluster.name: "{{ node_name }}", - discovery.zen.ping.unicast.hosts: "{{ groups['log-es-2'] }}", + cluster.name: "{{ log_es_etc_cluster_name }}", + discovery.zen.ping.unicast.hosts: "{{ groups['log-es'] }}", http.port: 9200, transport.tcp.port: 9300, node.data: "{{ es_etc_node_data | default('true') }}", node.master: "{{ es_etc_node_master | default('true') }}", bootstrap.memory_lock: true, }, - es_heap_size: "2g", - es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['log-es-2'] }}", + es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['log-es'] }}", es_etc_cluster_name: "{{ node_name }}" } + post_tasks: + # This will make some performance tuning + # Ref: https://aws.amazon.com/premiumsupport/knowledge-center/elasticsearch-indexing-performance/ + # + # Number of replicas: 0 # this is make sure that there is no backup and hence, no disk write. Good for logging indices + # refresh_interval: 60s # Refresh interval is the setting which make sure the latest indexed data will be available for search. + # # But refresh is a heavy resource intensive operation, which can be run once in a minute than once in a second + # # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html + # + - name: Changing log es index replica to zero + uri: + url: "http://{{groups['log-es'][0]}}:9200/_template/logstash" + body: |- + { + "index_patterns": [ + "{{log_es_index_pattern|d('logstash-*')}}" + ], + "settings": { + "number_of_shards": 5, + "number_of_replicas": 0, + "refresh_interval" : "60s" + } + } + body_format: json + headers: + Content-Type: application/json + http_agent: ansible-httpget + method: PUT + return_content: true + timeout: 30 diff --git a/ansible/firebase_deploy.yml b/ansible/firebase_deploy.yml new file mode 100644 index 000000000..4b411b4bd --- /dev/null +++ b/ansible/firebase_deploy.yml @@ -0,0 +1,6 @@ +--- +- hosts: localhost + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - firebase_deploy diff --git a/ansible/generic_templates/keycloak_301_rollback.sql b/ansible/generic_templates/keycloak_301_rollback.sql new file mode 100644 index 000000000..ddb61d380 --- /dev/null +++ b/ansible/generic_templates/keycloak_301_rollback.sql @@ -0,0 +1,35 @@ +ALTER TABLE REALM_SUPPORTED_LOCALES ADD COLUMN id SERIAL; +select count(*) from public.REALM_SUPPORTED_LOCALES; +DELETE FROM REALM_SUPPORTED_LOCALES a USING REALM_SUPPORTED_LOCALES b WHERE a.id < b.id AND a.value = b.value; +select count(*) from public.REALM_SUPPORTED_LOCALES; +ALTER TABLE REALM_SUPPORTED_LOCALES DROP COLUMN id; +ALTER TABLE composite_role ADD COLUMN id SERIAL; +select count(*) from public.composite_role; +DELETE FROM composite_role a USING composite_role b WHERE a.id < b.id AND a.composite = b.composite AND a.child_role = b.child_role; +select count(*) from public.composite_role; +ALTER TABLE composite_role DROP COLUMN id; +ALTER TABLE realm_events_listeners ADD COLUMN id SERIAL; +select count(*) from public.realm_events_listeners; +DELETE FROM realm_events_listeners a USING realm_events_listeners b WHERE a.id < b.id AND a.realm_id = b.realm_id AND a.value = b.value; +select count(*) from public.realm_events_listeners; +ALTER TABLE realm_events_listeners DROP COLUMN id; +ALTER TABLE redirect_uris ADD COLUMN id SERIAL; +select count(*) from public.redirect_uris; +DELETE FROM redirect_uris a USING redirect_uris b WHERE a.id < b.id AND a.client_id = b.client_id AND a.value = b.value; +select count(*) from public.redirect_uris; +ALTER TABLE redirect_uris DROP COLUMN id; +ALTER TABLE web_origins ADD COLUMN id SERIAL; +select count(*) from public.web_origins; +DELETE FROM web_origins a USING web_origins b WHERE a.id < b.id AND a.client_id = b.client_id AND a.value = b.value; +select count(*) from public.web_origins; +ALTER TABLE web_origins DROP COLUMN id; +ALTER TABLE databasechangelog ADD COLUMN serid SERIAL; +select count(*) from public.databasechangelog; +DELETE FROM databasechangelog a USING databasechangelog b WHERE a.serid < b.serid AND a.author = b.author AND a.md5sum = b.md5sum; +select count(*) from public.databasechangelog; +ALTER TABLE databasechangelog DROP COLUMN serid; +ALTER TABLE public.USER_ENTITY DROP COLUMN NOT_BEFORE; +select count(*) from offline_client_session; +select count(*) from offline_user_session; +delete from offline_user_session; +delete from offline_client_session; diff --git a/ansible/grafana-backup.yml b/ansible/grafana-backup.yml index bdac7e0e7..977087581 100644 --- a/ansible/grafana-backup.yml +++ b/ansible/grafana-backup.yml @@ -3,7 +3,7 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_name }}" roles: - grafana-backup diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 59d1e7023..445a29e68 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -3,7 +3,7 @@ # ENVIRONMENT CONFIGURATION proxy_server_name: "{{domain_name}}" keycloak_auth_server_url: "{{proto}}://{{ proxy_server_name }}/auth" -sunbird_echo_api_url: "{{proto}}://{{ proxy_server_name }}/api/echo/" +sunbird_echo_api_url: "http://kong:8000/echo/" sunbird_web_url: "{{proto}}://{{ proxy_server_name }}" sunbird_dataservice_url: "{{proto}}://{{ proxy_server_name }}/api/" api_proxy_name: "api.{{proxy_server_name}}" # Domain name on which device register and other api's will be accessed @@ -16,16 +16,11 @@ sunbird_instance: "{{env}}" env_short_name: "{{env}}" sunbird_env: "{{env}}" #Ekstep environment to connect to. Use `qa` for non-prod deployments, and `prod` for prod deployment. sunbird_app_name: "{{env}}" -azure_account_key: "{{core_vault_sunbird_azure_storage_key}}" #artifact upload -artifact_azure_account_name: "{{azure_account_name}}" -artifact_azure_account_key: "{{core_vault_sunbird_azure_storage_key}}" #plugins -azure_plugin_storage_account_name: "{{azure_account_name}}" -azure_plugin_storage_account_key: "{{core_vault_sunbird_azure_storage_key}}" # Keycloak keycloak_api_management_user_first_name: "admin" @@ -44,7 +39,7 @@ sunbird_keycloak_required_action_link_expiration_seconds: 2592000 sunbird_es_port: 9300 mail_server_port: 587 -upstream_url: "{{azure_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}" +upstream_url: "{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}" ## Mailing list # !! Override for specific alert groups; Default will be alerts_mailing_list @@ -85,10 +80,7 @@ kong_postgres_password: "{{core_vault_postgres_password}}" enc_postgres_user: "{{core_vault_postgres_username}}" badger_postgres_user: "{{core_vault_postgres_username}}" user_org_service_postgres_user: "{{core_vault_postgres_username}}" -sunbird_account_name: "{{azure_account_name}}" -sunbird_account_key: "{{core_vault_sunbird_azure_storage_key}}" ansible_vault_password: "{{ core_vault_ansible_vault_password }}" -sunbird_azure_account_name: "{{azure_account_name}}" jenkins_admin_username: "{{core_vault_jenkins_admin_username}}" jenkins_admin_password: "{{core_vault_jenkins_admin_password}}" vault_badging_authorization_key: "{{core_vault_badging_authorization_key}}" @@ -119,7 +111,7 @@ sunbird_es_host: "{{groups['es']| join(',')}}" ## Application server configurations sunbird_analytics_api_base_url: "http://{{analyticsapi_ip}}:9000" -sunbird_search_service_api_base_url: "http://{{searchservice_ip}}:9000" +sunbird_search_service_api_base_url: "http://{{sunbird_swarm_manager_lb_ip}}:9004" sunbird_mail_server_host: "{{ mail_server_host }}" #SMTP server IP. Could be ignored if not mails are to be sent. sunbird_mail_server_port: "{{ mail_server_port }}" #SMTP port. Could be ignored if not mails are to be sent. sunbird_mail_server_username: "{{ mail_server_username }}" #SMTP username. Could be ignored if not mails are to be sent. @@ -160,15 +152,14 @@ cassandra_port: 9042 cassandra_rpc_address: 0.0.0.0 cassandra_restore_dir: /home/deployer/ cassandra_backup_azure_container_name: cassandra-backup -cassandra_backup_azure_storage_account_name: "{{azure_account_name}}" -cassandra_backup_azure_storage_access_key: "{{backup_storage_key}}" cassandra_backup_dir: /data/cassandra/backup keycloak_realm: sunbird sunbird_content_player_url: "http://kong:8000/" sunbird_learner_player_url: "http://kong:8000/" -sunbird_sso_client_id: admin-cli +sunbird_sso_client_id: lms +sunbird_sso_client_secret: "{{core_vault_sunbird_sso_client_secret}}" sunbird_portal_realm: sunbird sunbird_portal_auth_server_client: portal sunbird_trampoline_client_id: trampoline @@ -186,8 +177,6 @@ sunbird_keycloak_public: true sunbird_cache_store: "memory" sunbird_portal_title_name: "{{sunbird_app_name}}" sunbird_sso_publickey: "{{core_vault_sunbird_sso_publickey}}" -sunbird_azure_storage_account: "{{azure_account_name}}" -sunbird_azure_storage_key: "{{core_vault_sunbird_azure_storage_key}}" sunbird_fcm_account_key: "{{core_vault_sunbird_fcm_account_key}}" sunbird_msg_91_auth: "{{core_vault_msg_91_auth_key}}" sunbird_telemetry_pdata_id: "{{env}}.sunbird.learning.service" @@ -205,9 +194,12 @@ sunbird_content_repo_api_key: "{{ core_vault_sunbird_ekstep_api_key }}" sunbird_search_service_api_key: "{{ core_vault_sunbird_ekstep_api_key }}" sunbird_dial_repo_api_base_url: "{{sunbird_ekstep_api_base_url}}" sunbird_dial_repo_api_key: "{{ core_vault_sunbird_ekstep_api_key }}" -sunbird_plugin_repo_api_base_url: "http://{{searchservice_ip}}:9000" +sunbird_plugin_repo_api_base_url: "{{ sunbird_search_service_api_base_url }}" sunbird_data_service_api_base_url: "{{sunbird_ekstep_api_base_url}}" sunbird_data_service_api_key: "{{ core_vault_sunbird_ekstep_api_key }}" +sunbird_content_service_api_base_url: "http://{{sunbird_swarm_manager_lb_ip}}:9002" +sunbird_user_service_api_base_url: "http://{{sunbird_swarm_manager_lb_ip}}:9000" +sunbird_group_service_api_base_url: "http://groups-service:9000" #API Manager kong_postgres_port: 5432 @@ -273,10 +265,6 @@ sunbird_cs_base_url: "http://{{sunbird_swarm_manager_lb_ip}}:5000" sunbird_user_service_base_url: "http://{{sunbird_swarm_manager_lb_ip}}:9000" kong_admin_api_url: http://localhost:8001 -postgresql_backup_azure_storage_account_name: "{{azure_account_name}}" -postgresql_backup_azure_storage_access_key: "{{ backup_storage_key }}" -postgresql_restore_azure_storage_account_name: "{{azure_account_name}}" -postgresql_restore_azure_storage_access_key: "{{ backup_storage_key }}" grafana_url: "{{proto}}://{{proxy_server_name}}/grafana" grafana_editor_username: editor @@ -325,9 +313,7 @@ sunbird_portal_cdn_url: "" sunbird_tenant_cdn_url: "" # sunbird_cdn_url -player_cdn_account_key: "{{core_vault_sunbird_azure_storage_key}}" player_cdn_container: player -player_cdn_account: "{{azure_account_name}}" #Additional Variables @@ -347,7 +333,6 @@ kong_version: 1.5.0-gold # Data pipeline vars learning_analytics_service_url: 'http://{{learningservice_ip}}:9000' -ep_es_host: "{{ groups['telemetry-search-cluster'][0] }}" #For kibana pipeline-dashboard #encryption Service @@ -364,7 +349,6 @@ sunbird_qrimage_topic: "{{env}}.qrimage.request" sunbird_azure_report_container_name: reports -sunbird_azure_account_key: "{{core_vault_sunbird_azure_storage_key}}" sunbird_google_captcha_site_key: "{{core_vault_sunbird_google_captcha_site_key_portal}}" sunbird_google_oauth_clientId: "{{core_vault_sunbird_google_oauth_clientId_portal}}" @@ -489,7 +473,7 @@ content_service_blacklisted_resourcetype: '' content_service_whitelisted_resourcetype: '' content_service_whitelisted_mimetype: '' content_service_blacklisted_mimetype: '' -sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,https://{{azure_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/' +sunbird_cloud_storage_urls: 'https://s3.ap-south-1.amazonaws.com/ekstep-public-{{ekstep_s3_env}}/,https://ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com/,https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{sunbird_content_azure_storage_container}}/' sunbird_email_max_recipients_limit: 100 sunbird_cassandra_consistency_level: one sunbird_cassandra_replication_strategy: '{"class":"SimpleStrategy","replication_factor":"1"}' @@ -501,76 +485,8 @@ sunbird_otp_expiration: 1800 sunbird_otp_length: 6 sunbird_help_link_visibility: FALSE - - -#Container Limits -proxy_replicas: 1 -proxy_reservation_memory: 64M -proxy_limit_memory: 128M -kong_replicas: 1 -kong_reservation_memory: 64M -kong_limit_memory: 256M -echo_service_replicas: 1 -echo_service_reservation_memory: 8M -echo_service_limit_memory: 16M -adminutil_replicas: 1 -adminutil_reservation_memory: 300M -adminutil_limit_memory: 300M -learner_limit_cpu: 1 -learner_replicas: 1 -learner_reservation_memory: 1000M -learner_limit_memory: 1000M -logger_logstash_heap_size: 512M -logger_logstash_replicas: 1 -logger_logstash_reservation_memory: 512M -logger_logstash_limit_memory: 750M -logger_kibana_reservation_memory: 750M -logger_kibana_limit_memory: 750M -logger_logspout_reservation_memory: 200M -logger_logspout_limit_memory: 200M -logger_oauth_reservation_memory: 32M -logger_oauth_limit_memory: 32M -prometheus_reservation_memory: 1G -prometheus_limit_memory: 1G -alertmanager_reservation_memory: 100M -alertmanager_limit_memory: 100M -node_exporter_reservation_memory: 16M -node_exporter_limit_memory: 32M -cadvisor_reservation_memory: 100M -cadvisor_limit_memory: 100M -elasticsearch_exporter_reservation_memory: 8M -elasticsearch_exporter_limit_memory: 24M -postgres_exporter_reservation_memory: 16M -postgres_exporter_limit_memory: 32M -statsd_exporter_reservation_memory: 8M -statsd_exporter_limit_memory: 16M -blackbox_exporter_reservation_memory: 16M -blackbox_exporter_limit_memory: 32M -jsonpath_exporter_reservation_memory: 32M -jsonpath_exporter_limit_memory: 64M -azure_blob_exporter_reservation_memory: 16M -azure_blob_exporter_limit_memory: 64M -grafana_reservation_memory: 100M -grafana_limit_memory: 100M -monitor_logstash_replicas: 1 -monitor_logstash_heap_size: 256m -monitor_logstash_reservation_memory: 256M -monitor_logstash_limit_memory: 350M -monitor_logspout_reservation_memory: 150M -monitor_logspout_limit_memory: 150M -player_replicas: 1 -player_reservation_memory: 256M -player_limit_memory: 512M -content_replicas: 1 -content_reservation_memory: 1000M -content_limit_memory: 1000M -badger_replicas: 1 -badger_reservation_memory: 500MB -badger_limit_memory: 500MB - - # not required -sunbird_image_storage_url: "https://{{azure_account_name}}.blob.core.windows.net/dial/" +sunbird_image_storage_url: "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/dial/" vault_auth_key: "{{core_vault_auth_key}}" vault_sender: "{{core_vault_sender}}" vault_country: "{{core_vault_country}}" @@ -594,8 +510,8 @@ config_service_enabled: false config_cache_ttl: 600 #### backup storage secret -backup_azure_storage_access_key: "{{core_vault_sunbird_azure_storage_key}}" -backup_azure_storage_account_name: "{{azure_account_name}}" +#backup_azure_storage_access_key: "{{core_vault_sunbird_azure_storage_key}}" +#backup_azure_storage_account_name: "{{azure_account_name}}" #es-backup app_es_snapshot_host: "{{ groups['es'][0] }}" @@ -613,7 +529,7 @@ sunbird_health_check_enable: 'true' ## Release 1.15 ## sunbird_keycloak_user_federation_provider_id: "{{core_vault_sunbird_keycloak_user_federation_provider_id}}" # Learner-service -sunbird_course_metrics_base_url: https://{{dp_azure_account_name}}.blob.core.windows.net/ +sunbird_course_metrics_base_url: https://{{sunbird_private_storage_account_name}}.blob.core.windows.net/ sunbird_gzip_size_threshold: 262144 prometheus_mount_point: "/root/dockerdata/prometheus/data/" @@ -634,8 +550,8 @@ sunbird_redis_host: "{{ groups['lp-redis'][0] }}" ### Release 2.1.0 ### sunbird_portal_offline_tenant: "" sunbird_portal_offline_supported_languages: "English" -sunbird_portal_offline_app_release_date: "" -sunbird_portal_offline_app_version: "1.0.0" +sunbird_portal_offline_app_release_date: "17/3/2020" +sunbird_portal_offline_app_version: "1.2.0" sunbird_portal_offline_app_download_url: "" sunbird_portal_log_level: "debug" @@ -654,7 +570,8 @@ enc_n_keys: 50 # Total number of keys required for encryption enc_n_reserved_keys: 15 # Number of keys which app reserves for usage kafka_topics_instruction: "{{env_name}}.coursebatch.job.request" kafka_urls: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" -kafka_topics_certificate_instruction: "{{env_name}}.coursebatch.certificate.request" +kafka_topics_certificate_instruction: "{{env_name}}.issue.certificate.request" +kafka_topics_contentstate_invalid: "{{env_name}}.contentstate.invalid" cert_service_container_name: "{{env}}-e-credentials" cert_service_cloud_storage_type: "{{cert_service_cloud_storage_type}}" @@ -675,20 +592,61 @@ api_report_mailing_list: "" ## This mailing list to send the daily api count rep kafka_assessment_topic: "{{env_name}}.telemetry.assess" # Portal vars for reports -sunbird_portal_azure_storage_account: "{{sunbird_azure_storage_account}}" -sunbird_portal_azure_storage_key: "{{sunbird_azure_storage_key}}" +#sunbird_portal_azure_storage_account: "{{sunbird_azure_storage_account}}" +#sunbird_portal_azure_storage_key: "{{sunbird_azure_storage_key}}" sunbird_device_api: "{{proto}}://{{ proxy_server_name }}/api/" sunbird_quartz_shadow_user_migration_timer: "0 0 1 1/1 * ? *" # Learning-Service -lp_cassandra_connection: "{{groups['cassandra']|join(':9042,')}}:9042" -dp_cassandra_connection: "{{ groups['dp-cassandra'][0] }}:9042" -environment_id: 10000000 -graph_passport_key: abc123 +lp_cassandra_connection: "{{groups['lp-cassandra']|join(':9042,')}}:9042" +dp_cassandra_connection: "{{groups['dp-cassandra']|join(':9042,')}}:9042" learning_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 language_neo4j_bolt_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 learning_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 learning_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:7687 language_read_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 language_write_elb_url: bolt://{{ groups['learning-neo4j-node1'][0] }}:8687 -mw_shard_id: 1 \ No newline at end of file +mw_shard_id: 1 +sunbird_lp_redis_host: "{{groups['lp-redis-ps'][0]}}" +#lp_azure_account_name: "{{azure_account_name}}" + +#Druid Proxy APi service +druid_proxy_replicas: 1 +druid_proxy_reservation_memory: 300M +druid_proxy_limit_memory: 300M +sunbird_druid_broker_host: "http://{{groups['dp-druid-broker'][0]}}" + +# Content-service +lp_cassandra_keyspace_prefix: "{{ env }}" +lp_kafka_url: "{{sunbird_processing_kafka_host}}" +content_import_required_props: '["name","code","mimeType","primaryCategory","artifactUrl","framework"]' +content_import_remove_props: '["downloadUrl","variants","previewUrl","streamingUrl","itemSets","level1Name","level1Concept","level2Name","level2Concept","level3Name","level3Concept","me_totalPlaySessionCount","me_totalTimeSpentInSec","me_totalSessionsCount","me_totalTimespent","me_totalInteractions","me_creationSessions","me_creationTimespent","me_averageInteractionsPerMin","me_averageSessionsPerDevice","me_totalDevices","me_averageTimespentPerSession","me_averageRating","me_totalDownloads","me_totalSideloads","me_totalRatings","me_totalComments","me_totalDialcode","me_totalDialcodeLinkedToContent","me_totalDialcodeAttached","me_hierarchyLevel","origin","originData","contentPolicyCheck","questions"]' + +# Print Service +#kp_sunbird_account_name: "{{azure_account_name}}" +#kp_sunbird_account_key: "{{core_vault_sunbird_azure_storage_key}}" + +#Sunbird-Portal release-2.6.5 # +sunbird_portal_updateLoginTimeEnabled: false + +# Desktop app vars +#sunbird_offline_azure_storage_account: "" #added this var for adopter usecase +offline_installer_container_name: "" #added this var for adopter usecase +cloud_storage_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net" + +# Search-service +search_index_host: "{{ groups['composite-search-cluster']|join(':9200,')}}:9200" +compositesearch_index_name: "compositesearch" +sunbird_report_service_url: "http://kong:8000/data/v1/report-service" + + +# Datapipeline +dp_play_http_secret_key: "mysecretdpplaysecretkey" + +# LMS Service vars +group_activity_agg_cache_ttl: 3600 +group_activity_agg_cache_enable: false + +# nodebb variables +sunbird_nodebb_storage_key: "{{ core_vault_sunbird_nodebb_storage_key }}" +mongo_nodebb_host: "{{ groups['mongo'] | join(':27017,')}}:27017" diff --git a/ansible/jenkins-backup.yml b/ansible/jenkins-backup.yml index 6643e8acb..4506a6bf6 100644 --- a/ansible/jenkins-backup.yml +++ b/ansible/jenkins-backup.yml @@ -4,8 +4,8 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_storage_name }}" - AZURE_STORAGE_KEY: "{{ backup_storage_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - jenkins-backup-upload diff --git a/ansible/keycloak-migration.yaml b/ansible/keycloak-migration.yaml new file mode 100644 index 000000000..ac465c1f8 --- /dev/null +++ b/ansible/keycloak-migration.yaml @@ -0,0 +1,197 @@ +--- +# This ansible playbook will do keycloak migration from 3.0.1 to 6.0.1 +# Prerequisites +# 1. Keycloak build artifact with 6.0.1 +# 2. Creates /opt/keycloak_new directory for migration +# 3. Create /opt/keyloak_old with current keycloak backup +# +# Ref: https://project-sunbird.atlassian.net/wiki/spaces/UM/pages/1087504491/KeyCloak+6.0.1+upgrade + +- hosts: keycloak + gather_facts: yes + vars: + keycloak_dir: /opt/keycloak + vars_files: + - "{{ inventory_dir }}/secrets.yml" + tasks: + - name: Get the list of all the services running in our OS + become: yes + service_facts: + + - name: Stop the monit to ensure that it doesn't start keycloak + service: name=monit state=stopped + become: yes + when: ansible_facts.services.monit is defined + + - name: stop the keycloak server so that it will not cause any issues at time of migration + become: yes + shell: /bin/bash -c "/etc/init.d/keycloak stop || pkill keycloak" + failed_when: false + +- hosts: local + vars_files: + - "{{ inventory_dir }}/secrets.yml" + vars: + db_backup_name: "/root/keycloak{{ ansible_date_time.epoch }}.sql" + no_log: true + tasks: + - name: taking backup of existing DB + become: yes + postgresql_db: + name: "{{ keycloak_postgres_database }}" + state: dump + target: "{{ db_backup_name }}" + login_user: "{{keycloak_postgres_user }}" + login_password: "{{ keycloak_postgres_password }}" + login_host: "{{ keycloak_postgres_host }}" + port: 5432 + + - name: creating backup db "{{ keycloak_postgres_database }}_301" + postgresql_db: + name: "{{ keycloak_postgres_database }}_301" + state: present + login_user: "{{ keycloak_postgres_user }}" + login_password: "{{ keycloak_postgres_password }}" + login_host: "{{ keycloak_postgres_host }}" + register: status + + - name: creating backup keycloak old db {{ keycloak_postgres_database }}_301 for the first time + become: yes + postgresql_db: + name: "{{ keycloak_postgres_database }}_301" + state: restore + target: "{{ db_backup_name }}" + login_user: "{{ keycloak_postgres_user }}" + login_password: "{{ keycloak_postgres_password }}" + login_host: "{{ keycloak_postgres_host }}" + port: 5432 + when: status.changed + + - name: copying keycloak db revert script. + copy: + src: generic_templates/keycloak_301_rollback.sql + dest: /tmp/keycloak_301_rollback.sql + + - name: Altering the schema to delete duplicate events/entries from the keycloak DB + become: yes + postgresql_db: + name: "{{ keycloak_postgres_database }}" + state: restore + target: /tmp/keycloak_301_rollback.sql + login_user: "{{keycloak_postgres_user }}" + login_password: "{{ keycloak_postgres_password }}" + login_host: "{{ keycloak_postgres_host }}" + port: 5432 + +- hosts: keycloak + vars: + keycloak_dir: /opt/keycloak + vars_files: + - "{{ inventory_dir }}/secrets.yml" + tasks: + - name: Stopping keycloak + become: yes + shell: /bin/bash -c "/etc/init.d/keycloak stop || pkill keycloak" + failed_when: false + + - name: Waiting for keycloak to stop completely + wait_for: + port: 8080 + state: stopped + timeout: 30 + + - name: deleting keycloak new and old directory if present + become: yes + file: + path: "{{ item }}" + state: absent + force: yes + with_items: + - "{{ keycloak_dir }}_new" + - "{{ keycloak_dir }}_old" + +# Ansible copy module doesn't copy directories recursively in remote servers so using shell module + - name: Create backup of existing keycloak 3.2.0 version + become: yes + shell: "cp -rf {{ keycloak_dir }} {{ keycloak_dir }}_old && chown -R keycloak:keycloak {{ keycloak_dir }}_old" + + - name: make sure keycloak path exists + become: yes + file: + path: "{{ keycloak_dir }}_new" + state: directory + owner: keycloak + group: keycloak + + - name: unarchive keycloak + become: yes + unarchive: + src: https://sunbirdpublic.blob.core.windows.net/installation/keycloak-6.0.1.tar.gz + dest: "{{ keycloak_dir }}_new" + extra_opts: ['--strip-components=1'] + remote_src: yes + owner: keycloak + group: keycloak + + - name: Copying old files to new keycloak for migration + become: yes + become_user: keycloak + shell: "cp -rf {{ keycloak_dir }}_old/{{ item }}/* {{ keycloak_dir }}_new/{{ item }}/" + with_items: + - standalone + - domain + + - name: Copying old files to new keycloak for migration + become: yes + become_user: keycloak + shell: "cp -rf {{ keycloak_dir }}_old/{{ item }} {{ keycloak_dir }}_new/{{ item }}" + with_items: + - modules/system/layers/keycloak/org/postgresql + + - name: Deleting unnecessary fields from standalone.xml file + become: yes + become_user: keycloak + lineinfile: + path: "{{keycloak_dir }}_new/standalone/configuration/standalone-ha.xml" + state: absent + regexp: "{{ item }}" + with_items: + - '.*' + - '.*' + - '.*' + + - name: Running migration + become: yes + become_user: keycloak + shell: "bash {{ item }}" + with_items: + - "bin/jboss-cli.sh --file=bin/migrate-standalone.cli" + - "bin/jboss-cli.sh --file=bin/migrate-standalone-ha.cli" + args: + chdir: "{{ keycloak_dir }}_new" + run_once: yes + + - name: delete the old keycloak version from the server + become: yes + file: + path: "{{ keycloak_dir }}" + state: absent + force: yes + + - name: renaming the folder to make version 6 as the keycloak version + become: yes + shell: "mv {{ keycloak_dir }}_new {{ keycloak_dir }} && chown -R keycloak:keycloak {{ keycloak_dir }}" + + - name: Starting keycloak in both the servers + become: yes + shell: "/etc/init.d/keycloak start" + + - name: get the list of all services in the system + become: yes + service_facts: + + - name: After succesful migration and staring of keycloak, start back monit if installed + service: name=monit state=started + become: yes + when: ansible_facts.services.monit is defined + diff --git a/ansible/keycloak.yml b/ansible/keycloak.yml index 3ec94fd47..54e248903 100644 --- a/ansible/keycloak.yml +++ b/ansible/keycloak.yml @@ -3,23 +3,11 @@ become: true vars_files: - ['{{ inventory_dir }}/secrets.yml', 'secrets/{{ env }}.yml'] - # --tags provision/deployment will trigger respective roles - # --tags bootstrap will trigger user/realm creation - pre_tasks: - - name: Get the list of all the services running in our OS - become: yes - service_facts: - - - name: Stop the monit to ensure that it doesn't start keycloak - service: name=monit state=stopped - become: yes - when: ansible_facts.services.monit is defined - roles: - {role: openjdk, tags: provision} - {role: keycloak-provision, tags: provision} - keycloak-deploy - - {role: monit, monit_checks: ['keycloak'], tags: deploy} + # - {role: monit, monit_checks: ['keycloak'], tags: deploy} post_tasks: - name: get the list of all services in the system @@ -29,4 +17,4 @@ - name: After succesful migration and staring of keycloak, start back monit if installed service: name=monit state=started become: yes - when: ansible_facts.services.monit is defined \ No newline at end of file + when: ansible_facts.services.monit is defined diff --git a/ansible/kp_upload-schema.yml b/ansible/kp_upload-schema.yml new file mode 100644 index 000000000..0ca52f5f0 --- /dev/null +++ b/ansible/kp_upload-schema.yml @@ -0,0 +1,13 @@ +- hosts: local + become: yes + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_public_storage_account_sas }}" + tasks: + - name: upload batch + command: "az storage blob upload-batch --destination {{ plugin_container_name }}/schemas/local --source {{ source_name }}" + async: 3600 + poll: 10 \ No newline at end of file diff --git a/ansible/log_es_restore.yml b/ansible/log_es_restore.yml new file mode 100644 index 000000000..741ad4ce2 --- /dev/null +++ b/ansible/log_es_restore.yml @@ -0,0 +1,10 @@ +--- +- hosts: "{{ es_restore_host }}" + become: yes + vars_files: + - ['{{inventory_dir}}/secrets.yml'] + roles: + - log-es-azure-restore + tags: + - es_restore + diff --git a/ansible/logging.yml b/ansible/logging.yml index ca8beb0e1..fd3dc74c9 100644 --- a/ansible/logging.yml +++ b/ansible/logging.yml @@ -1,9 +1,12 @@ -- name: Spray cluster name to nodes - hosts: swarm-nodes +--- +- hosts: swarm-nodes become: yes tasks: - - copy: dest=/home/deployer/cluster_name content="{{ cluster_name }}" + - name: Spray cluster name to nodes + copy: dest=/home/deployer/cluster_name content="{{ cluster_name }}" when: cluster_name is defined + tags: + - stack-logger - hosts: swarm-bootstrap-manager become: yes diff --git a/ansible/mobile_upload.yml b/ansible/mobile_upload.yml new file mode 100644 index 000000000..35c93673e --- /dev/null +++ b/ansible/mobile_upload.yml @@ -0,0 +1,6 @@ +--- +- hosts: localhost + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - mobile_upload diff --git a/ansible/mongodb-cluster.yml b/ansible/mongodb-cluster.yml new file mode 100644 index 000000000..81cab6a5f --- /dev/null +++ b/ansible/mongodb-cluster.yml @@ -0,0 +1,13 @@ +- hosts: mongo + become: yes + vars_files: + - "{{inventory_dir}}/secrets.yml" + vars: + mongodb_replication_params: + mongodb_conf_auth: true + mongodb_conf_replSet: mongors + mongodb_conf_dbpath: /var/lib/mongodb + mongodb_conf_bind_ip: 0.0.0.0 + mongodb_conf_port: 27017 + roles: + - mongodb-cluster diff --git a/ansible/nodebbui-upload.yml b/ansible/nodebbui-upload.yml new file mode 100644 index 000000000..92b484a58 --- /dev/null +++ b/ansible/nodebbui-upload.yml @@ -0,0 +1,19 @@ +- hosts: local + become: yes + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_public_storage_account_sas }}" + tasks: + - name: delete batch + shell: | + azcopy rm "https://{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ nodebbui_container_name }}{{sunbird_public_storage_account_sas}}" --recursive=true + async: 3600 + poll: 10 + + - name: upload batch + command: "az storage blob upload-batch --destination {{ nodebbui_container_name }} --source {{ source_name }}" + async: 3600 + poll: 10 diff --git a/ansible/offline-installer.yml b/ansible/offline-installer.yml index 568b9d67e..24e511c3d 100644 --- a/ansible/offline-installer.yml +++ b/ansible/offline-installer.yml @@ -1,7 +1,6 @@ --- - hosts: local - become: yes vars_files: - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml'] roles: - - offline-installer + - desktop-deploy diff --git a/ansible/plugin.yml b/ansible/plugin.yml index 2276d95b9..ac8f93b5c 100644 --- a/ansible/plugin.yml +++ b/ansible/plugin.yml @@ -4,8 +4,8 @@ vars_files: - "{{inventory_dir}}/secrets.yml" environment: - AZURE_STORAGE_ACCOUNT: "{{ azure_plugin_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ azure_plugin_storage_account_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_public_storage_account_key }}" tasks: - name: delte plugin org_sunbird_questionunit_quml command: "az storage blob delete-batch --source {{ plugin_container_name }} --pattern content-plugins/{{ plugins_name }}" diff --git a/ansible/prometheus-backup.yml b/ansible/prometheus-backup.yml index 74c790eb1..d31adbd12 100644 --- a/ansible/prometheus-backup.yml +++ b/ansible/prometheus-backup.yml @@ -7,8 +7,8 @@ vars_files: - ['{{ inventory_dir }}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-backup-v2 tags: @@ -23,8 +23,8 @@ vars_files: - ['{{inventory_dir}}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-backup-v2 tags: @@ -40,8 +40,8 @@ vars_files: - ['{{inventory_dir}}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-backup-v2 tags: diff --git a/ansible/prometheus-restore.yml b/ansible/prometheus-restore.yml index 6eeb5ccbc..5d4bba973 100644 --- a/ansible/prometheus-restore.yml +++ b/ansible/prometheus-restore.yml @@ -7,8 +7,8 @@ vars_files: - ['{{ inventory_dir }}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-restore tags: @@ -23,8 +23,8 @@ vars_files: - ['{{ inventory_dir }}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-restore tags: @@ -39,9 +39,9 @@ vars_files: - ['{{ inventory_dir }}/secrets.yml'] environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" roles: - prometheus-restore tags: - - statefull-prometheus-restore \ No newline at end of file + - statefull-prometheus-restore diff --git a/ansible/provision.yml b/ansible/provision.yml index 8fc0d305f..b85eeaad5 100644 --- a/ansible/provision.yml +++ b/ansible/provision.yml @@ -29,7 +29,6 @@ node.master: "{{ es_etc_node_master | default('true') }}", bootstrap.memory_lock: true, }, - es_heap_size: "2g", es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['es'] }}", es_etc_cluster_name: "{{ app_es_etc_cluster_name }}" } diff --git a/ansible/redis-backup.yml b/ansible/redis-backup.yml new file mode 100644 index 000000000..af7b1564e --- /dev/null +++ b/ansible/redis-backup.yml @@ -0,0 +1,11 @@ +- hosts: redisall + become: yes + gather_facts: false + vars_files: + - ['{{inventory_dir}}/secrets.yml'] + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" + roles: + - redis-backup + run_once: true diff --git a/ansible/roles/assets-upload-azure/tasks/main.yml b/ansible/roles/assets-upload-azure/tasks/main.yml index 398fb718b..520641f5e 100755 --- a/ansible/roles/assets-upload-azure/tasks/main.yml +++ b/ansible/roles/assets-upload-azure/tasks/main.yml @@ -2,8 +2,8 @@ - name: Deleting container before Uploding assets command: az storage blob delete-batch -s {{player_cdn_container}} environment: - AZURE_STORAGE_ACCOUNT: "{{player_cdn_account}}" - AZURE_STORAGE_KEY: "{{player_cdn_account_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_SAS_TOKEN: "{{sunbird_public_storage_account_sas}}" async: 3600 poll: 10 @@ -11,16 +11,15 @@ command: az storage container create --name {{player_cdn_container}} ignore_errors: true environment: - AZURE_STORAGE_ACCOUNT: "{{player_cdn_account}}" - AZURE_STORAGE_KEY: "{{player_cdn_account_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_SAS_TOKEN: "{{sunbird_public_storage_account_sas}}" # Upload the assets created by the job to azure - name: Upload to azure blob storage command: az storage blob upload-batch -d {{player_cdn_container}} -s {{assets}} environment: - AZURE_STORAGE_ACCOUNT: "{{player_cdn_account}}" - AZURE_STORAGE_KEY: "{{player_cdn_account_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_SAS_TOKEN: "{{sunbird_public_storage_account_sas}}" async: 3600 poll: 10 - diff --git a/ansible/roles/azure-cli/tasks/main.yml b/ansible/roles/azure-cli/tasks/main.yml index ae35cc848..484bf1f4f 100644 --- a/ansible/roles/azure-cli/tasks/main.yml +++ b/ansible/roles/azure-cli/tasks/main.yml @@ -1,15 +1,12 @@ -- name: Add Azure apt repository - apt_repository: repo='deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ xenial main' state=present - -- name: Import Azure signing key +- name: install azure cli become: yes - apt_key: - url: https://packages.microsoft.com/keys/microsoft.asc - state: present - #shell: curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - -- name: ensure azure-cli and apt-transport-https is installed - apt: name={{ item }} state=present update_cache=yes allow_unauthenticated=yes - with_items: - - apt-transport-https - - azure-cli + shell: + which az || curl -sL https://aka.ms/InstallAzureCLIDeb | bash +- name: install azcopy + become: yes + shell: | + which azcopy || ( \ + mkdir /tmp/azcopy && cd /tmp/azcopy && \ + wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1 \ + && mv azcopy /usr/local/bin \ + && rm -rf /tmp/azcopy ) diff --git a/ansible/roles/cassandra-backup/tasks/main.yml b/ansible/roles/cassandra-backup/tasks/main.yml index 9024811e5..c06685ca9 100755 --- a/ansible/roles/cassandra-backup/tasks/main.yml +++ b/ansible/roles/cassandra-backup/tasks/main.yml @@ -1,23 +1,27 @@ +- name: Make sure backup dir is empty + file: path="{{ cassandra_backup_dir }}" state=absent + ignore_errors: true + - name: Create the directory become: yes file: path=/data/cassandra/backup state=directory recurse=yes - name: copy the backup script become: yes - template: src=cassandra_backup.j2 dest=/data/cassandra/backup/cassandra_backup.py mode=0755 + template: + src: ../../../../deploy/cassandra_backup.py + dest: /data/cassandra/backup/cassandra_backup.py + mode: 0755 - set_fact: - cassandra_backup_gzip_file_name: "cassandra-backup-{{ lookup('pipe', 'date +%Y%m%d') }}-{{ ansible_hostname }}" - -- set_fact: - cassandra_backup_gzip_file_path: "{{ cassandra_backup_dir }}/{{ cassandra_backup_gzip_file_name }}.tar.gz" + cassandra_backup_gzip_file_name: "cassandra-backup-{{ lookup('pipe', 'date +%Y%m%d') }}-{{ ansible_hostname }}-new" - name: run the backup script become: yes - shell: python cassandra_backup.py --snapshotname "{{ cassandra_backup_gzip_file_name }}" + shell: python3 cassandra_backup.py --snapshotname "{{ cassandra_backup_gzip_file_name }}" --snapshotdirectory "{{ cassandra_backup_gzip_file_name }}" "{{additional_arguments|d('')}}" args: chdir: /data/cassandra/backup - async: 7200 + async: 14400 poll: 30 - name: Check doc_root path @@ -32,14 +36,15 @@ command: az storage container create --name {{ cassandra_backup_azure_container_name }} ignore_errors: true environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" - name: Upload to azure blob storage - command: az storage blob upload --name "{{ cassandra_backup_gzip_file_name }}".tar.gz --file {{ cassandra_backup_gzip_file_path }} --container-name {{ cassandra_backup_azure_container_name }} + command: "azcopy copy {{cassandra_backup_gzip_file_name}} 'https://{{sunbird_management_storage_account_name}}.blob.core.windows.net/{{cassandra_backup_azure_container_name}}{{sunbird_management_storage_account_sas}}' --recursive" environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZCOPY_CONCURRENT_FILES: 10 # How many files azcopy should read concurrently. + args: + chdir: /data/cassandra/backup async: 3600 poll: 10 diff --git a/ansible/roles/cassandra-db-update/tasks/main.yml b/ansible/roles/cassandra-db-update/tasks/main.yml new file mode 100644 index 000000000..8f01c22a2 --- /dev/null +++ b/ansible/roles/cassandra-db-update/tasks/main.yml @@ -0,0 +1,28 @@ +- set_fact: timestamp="cqlsh_{{ansible_date_time.epoch}}" + +- name: Make sure cassandra is up and listening + wait_for: + port: 9042 + timeout: 180 + sleep: 10 + +- name: Copy cql script + become: yes + template: + src: "{{cql_file}}" + dest: "/tmp/{{cql_file}}" + mode: 0755 + +- name: run cql + become: yes + shell: 'cqlsh -f /tmp/{{cql_file}}' + async: 3600 + poll: 10 + ignore_errors: true + +- name: remove cql file + become: yes + file: + path: "/tmp/{{cql_file}}" + state: absent + diff --git a/ansible/roles/cassandra-db-update/templates/alter_externalId_table.cql b/ansible/roles/cassandra-db-update/templates/alter_externalId_table.cql new file mode 100644 index 000000000..2deb083d0 --- /dev/null +++ b/ansible/roles/cassandra-db-update/templates/alter_externalId_table.cql @@ -0,0 +1,40 @@ +copy sunbird.usr_external_identity(provider, +idtype, +externalid, +createdby, +createdon, +lastupdatedby, +lastupdatedon, +originalexternalid, +originalidtype, +originalprovider, +userid) to 'usr_external_identity.csv' with header=true and numprocesses=8 and maxattempts=10; + +DROP TABLE IF EXISTS sunbird.usr_external_identity; + +CREATE TABLE sunbird.usr_external_identity( +externalId text, +provider text, +idType text, +userId text, +createdOn timestamp, +lastUpdatedOn timestamp, +createdBy text, +lastUpdatedBy text, +originalExternalId text, +originalIdType text, +originalProvider text, +PRIMARY KEY(userId, idType, provider)); + + +copy sunbird.usr_external_identity(provider, +idtype, +externalid, +createdby, +createdon, +lastupdatedby, +lastupdatedon, +originalexternalid, +originalidtype, +originalprovider, +userid) from 'usr_external_identity.csv' with header=true and chunksize=2000 and numprocesses=8; diff --git a/ansible/roles/cassandra-db-update/templates/alter_user_feed_table.cql b/ansible/roles/cassandra-db-update/templates/alter_user_feed_table.cql new file mode 100644 index 000000000..beba479d4 --- /dev/null +++ b/ansible/roles/cassandra-db-update/templates/alter_user_feed_table.cql @@ -0,0 +1,40 @@ +copy sunbird.user_feed( +id, +category, +createdby, +createdon, +data, +expireon, +priority, +status, +updatedby, +updatedon, +userid) to 'user_feed_{{timestamp}}.csv' with header=true and numprocesses=8 and maxattempts=10; + +DROP TABLE IF EXISTS sunbird.user_feed; + +CREATE TABLE sunbird.user_feed( +id text, +category text, +createdby text, +createdon timestamp, +data text, +expireon timestamp, +priority int, +status text, +updatedby text, +updatedon timestamp, +userid text, +PRIMARY KEY(userId, category, id)); + +copy sunbird.user_feed(id, +category, +createdby, +createdon, +data, +expireon, +priority, +status, +updatedby, +updatedon, +userid) from 'user_feed_{{timestamp}}.csv' with header=true and chunksize=2000 and numprocesses=8 and maxattempts=10; diff --git a/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml b/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml index c38c2c9d3..02efebd0e 100644 --- a/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml +++ b/ansible/roles/cassandra-deploy-decrypt/defaults/main.yml @@ -1,2 +1,3 @@ cassandra_jar_path: ../ cassandra_deploy_path: /home/deployer +cassandra_jar_name: user-declaration-util-1.0-SNAPSHOT-jar-with-dependencies.jar diff --git a/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml b/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml index c26748ff3..0275452cd 100644 --- a/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml +++ b/ansible/roles/cassandra-deploy-decrypt/tasks/main.yml @@ -1,10 +1,10 @@ - name: Copy the jar become: yes copy: - src: "{{cassandra_jar_path}}/decryptionUtil-1.0-SNAPSHOT-jar-with-dependencies.jar" + src: "{{cassandra_jar_path}}/{{cassandra_jar_name}}" dest: "{{cassandra_deploy_path}}" run_once: true - name: Run the application - shell: java -jar decryptionUtil-1.0-SNAPSHOT-jar-with-dependencies.jar + shell: "java -jar {{cassandra_jar_name}}" run_once: true diff --git a/ansible/roles/cassandra-deploy/defaults/main.yml b/ansible/roles/cassandra-deploy/defaults/main.yml index c38c2c9d3..387922175 100644 --- a/ansible/roles/cassandra-deploy/defaults/main.yml +++ b/ansible/roles/cassandra-deploy/defaults/main.yml @@ -1,2 +1 @@ cassandra_jar_path: ../ -cassandra_deploy_path: /home/deployer diff --git a/ansible/roles/cassandra-deploy/tasks/main.yml b/ansible/roles/cassandra-deploy/tasks/main.yml index 15a1090b6..9718fea2f 100644 --- a/ansible/roles/cassandra-deploy/tasks/main.yml +++ b/ansible/roles/cassandra-deploy/tasks/main.yml @@ -1,8 +1,15 @@ +- name: checking for last failed migration + shell: cqlsh -e 'SELECT version from sunbird.cassandra_migration_version where success=false ALLOW FILTERING ;' | tail -n +4 | head -n -2 | sort | tail -n1 | xargs + register: version + +- name: Dropping failed migration {{ version.stdout }} + shell: cqlsh -e "delete from sunbird.cassandra_migration_version where version='{{version.stdout}}';" + when: version.stdout + - name: Copy Data Products - become: yes copy: - src: "{{cassandra_jar_path}}/cassandra-migration-0.0.1-SNAPSHOT-jar-with-dependencies.jar" - dest: "{{cassandra_deploy_path}}" + src: "{{cassandra_jar_path}}cassandra-migration-0.0.1-SNAPSHOT-jar-with-dependencies.jar" + dest: "/home/{{ansible_ssh_user}}/" run_once: true - name: Run the application diff --git a/ansible/roles/cassandra-exporter/README.md b/ansible/roles/cassandra-exporter/README.md new file mode 100644 index 000000000..e388a8f39 --- /dev/null +++ b/ansible/roles/cassandra-exporter/README.md @@ -0,0 +1,43 @@ +Role Name +========= + +Role to install `cassandra exporter java binary` as a `systemd` service + +Role Variables +-------------- + +| Name | Default Value | Description | +| -------------- | ------------- | -----------------------------------| +| `cassandra_exporter_version` | 2.2.0 | Cassandra exporter package version | +| `cassandra_listen_address` | 0.0.0.0 | Cassandra exporter listening address | +| `cassandra_listen_port` | 8080 | Cassandra exporter listening port | +| `cassandra_exporter_binary_url` | https://github.com/criteo/cassandra_exporter/releases/download/{{cassandra_exporter_version}}/cassandra_exporter-{{cassandra_exporter_version}}-all.jar | Cassandra exporter jar download location | +| `cassandra_exporter_config_url` | https://raw.githubusercontent.com/criteo/cassandra_exporter/master/config.yml | Cassandra exporter config download location | +| `cassandra_exporter_config` | | Cassandra exporter config file in local disk, if defined ,`cassandra_exporter_config_url` is ignored | +| `cassandra_exporter_user` | cassandra-exp | UNIX user to run the binary | +| `cassandra_exporter_group` | cassandra-exp | UNIX group to run the binary | +| `cassandra_exporter_root_dir` | /opt/cassandra_exporter | Base location where cassandra exporter stuff is downloaded | +| `cassandra_exporter_dist_dir` | /opt/cassandra_exporter/dist | Location for binary and systemd service script | +| `cassandra_exporter_config_dir` | /opt/cassandra_exporter/config | Location for config | + + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + become: true + become_method: sudo + roles: + - role: ansible-cassandra-exporter + +Usage +----- + +You may install the role locally using `ansible-galaxy install -r requirements.yml -p roles/` where `requirements.yml` reads something like + +``` +- src: git+https://github.com/criteo/cassandra_exporter/ansible/roles + version: TBD +``` diff --git a/ansible/roles/cassandra-exporter/defaults/main.yml b/ansible/roles/cassandra-exporter/defaults/main.yml new file mode 100644 index 000000000..8a9309e8a --- /dev/null +++ b/ansible/roles/cassandra-exporter/defaults/main.yml @@ -0,0 +1,15 @@ +--- + +cassandra_exporter_version: 2.3.5 + +cassandra_exporter_listen_address: "0.0.0.0" +cassandra_exporter_listen_port: "8080" +cassandra_exporter_binary_url: 'https://github.com/criteo/cassandra_exporter/releases/download/{{cassandra_exporter_version}}/cassandra_exporter-{{cassandra_exporter_version}}.jar' +cassandra_exporter_config_url: 'https://raw.githubusercontent.com/criteo/cassandra_exporter/{{cassandra_exporter_version}}/config.yml' + +cassandra_exporter_user: cassandra-exp +cassandra_exporter_group: cassandra-exp + +cassandra_exporter_root_dir: /opt/cassandra_exporter +cassandra_exporter_dist_dir: "{{ cassandra_exporter_root_dir }}/dist" +cassandra_exporter_config_dir: "{{ cassandra_exporter_root_dir }}/config" diff --git a/ansible/roles/cassandra-exporter/handlers/main.yml b/ansible/roles/cassandra-exporter/handlers/main.yml new file mode 100644 index 000000000..6973d9f19 --- /dev/null +++ b/ansible/roles/cassandra-exporter/handlers/main.yml @@ -0,0 +1,15 @@ +--- + +- name: reenable cassandra exporter service + systemd: + daemon_reload: yes + enabled: yes + name: prometheus-cassandra-exporter + +- name: reinit cassandra exporter + command: initctl reload-configuration + +- name: restart cassandra exporter + service: + name: prometheus-cassandra-exporter + state: restarted diff --git a/ansible/roles/cassandra-exporter/meta/main.yml b/ansible/roles/cassandra-exporter/meta/main.yml new file mode 100644 index 000000000..326a6298c --- /dev/null +++ b/ansible/roles/cassandra-exporter/meta/main.yml @@ -0,0 +1,60 @@ +galaxy_info: + author: Pankaj M. Tolani + description: DevOps Engineer + company: Autonomous Thingz Pty Ltd + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 2.4 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: Cassandra, JMX, Exporter, Systemd, Service + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/ansible/roles/cassandra-exporter/tasks/main.yml b/ansible/roles/cassandra-exporter/tasks/main.yml new file mode 100644 index 000000000..b342701b4 --- /dev/null +++ b/ansible/roles/cassandra-exporter/tasks/main.yml @@ -0,0 +1,97 @@ +--- + +- name: 'create cassandra exporter group' + group: + name: '{{ cassandra_exporter_group }}' + system: yes + state: present + +- name: 'create cassandra exporter user' + user: + name: '{{ cassandra_exporter_user }}' + system: yes + shell: '/sbin/nologin' + group: '{{ cassandra_exporter_group }}' + createhome: no + +- name: 'create cassandra exporter directories' + file: + path: '{{ item }}' + state: directory + owner: '{{ cassandra_exporter_user }}' + group: '{{ cassandra_exporter_group }}' + mode: 0755 + with_items: + - '{{ cassandra_exporter_root_dir }}' + - '{{ cassandra_exporter_dist_dir }}' + - '{{ cassandra_exporter_config_dir }}' + +- name: 'download cassandra exporter binary' + get_url: + url: '{{ cassandra_exporter_binary_url }}' + dest: '{{ cassandra_exporter_dist_dir }}/cassandra.jar' + +- name: 'Copy cassandra exporter config' + template: + src: cassandra-exporter-config.yaml.j2 + dest: '{{ cassandra_exporter_config_dir }}/config.yml' + notify: + - restart cassandra exporter + +- name: 'set cassandra listen address' + lineinfile: + regexp: "^listenAddress" + dest: "{{ cassandra_exporter_config_dir }}/config.yml" + line: "listenAddress: {{ cassandra_exporter_listen_address }}" + notify: + - restart cassandra exporter + +- name: 'set cassandra listen port' + lineinfile: + regexp: "^listenPort" + dest: "{{ cassandra_exporter_config_dir }}/config.yml" + line: "listenPort: {{ cassandra_exporter_listen_port }}" + notify: + - restart cassandra exporter + +- name: 'generate the shell script' + template: + src: cassandra_exporter.sh.j2 + dest: '{{ cassandra_exporter_dist_dir }}/cassandra_exporter.sh' + owner: root + group: root + mode: 0755 + register: prometheus_cassandra_exporter_updated + notify: + - restart cassandra exporter + +- name: 'update group and owner for files' + file: + path: '{{ item }}' + state: file + owner: '{{ cassandra_exporter_user }}' + group: '{{ cassandra_exporter_group }}' + with_items: + - '{{ cassandra_exporter_dist_dir }}/cassandra.jar' + - '{{ cassandra_exporter_config_dir }}/config.yml' + when: prometheus_cassandra_exporter_updated is changed + +- name: 'create systemd service unit' + template: + src: etc/systemd/system/prometheus-cassandra-exporter.service.j2 + dest: /etc/systemd/system/prometheus-cassandra-exporter.service + owner: root + group: root + mode: 0644 + when: service_mgr | default(ansible_service_mgr) == 'systemd' + notify: + - reenable cassandra exporter service + - restart cassandra exporter + +- meta: flush_handlers + +- name: 'ensure prometheus cassandra exporter service is enabled and started' + service: + name: prometheus-cassandra-exporter + state: started + enabled: yes diff --git a/ansible/roles/cassandra-exporter/templates/cassandra-exporter-config.yaml.j2 b/ansible/roles/cassandra-exporter/templates/cassandra-exporter-config.yaml.j2 new file mode 100644 index 000000000..f5d6b595f --- /dev/null +++ b/ansible/roles/cassandra-exporter/templates/cassandra-exporter-config.yaml.j2 @@ -0,0 +1,139 @@ +host: localhost:7199 +ssl: False +user: +password: +listenAddress: 0.0.0.0 +listenPort: 8080 +# Regular expression to match environment variable names that will be added +# as labels to all data points. The name of the label will be either +# $1 from the regex below, or the entire environment variable name if no match groups are defined +# +# Example: +# additionalLabelsFromEnvvars: "^ADDL\_(.*)$" +additionalLabelsFromEnvvars: +blacklist: + # To profile the duration of jmx call you can start the program with the following options + # > java -Dorg.slf4j.simpleLogger.defaultLogLevel=trace -jar cassandra_exporter.jar config.yml --oneshot + # + # To get intuition of what is done by cassandra when something is called you can look in cassandra + # https://github.com/apache/cassandra/tree/trunk/src/java/org/apache/cassandra/metrics + # Please avoid to scrape frequently those calls that are iterating over all sstables + + # Unaccessible metrics (not enough privilege) + - java:lang:memorypool:.*usagethreshold.* + + # Leaf attributes not interesting for us but that are presents in many path + - .*:999thpercentile + - .*:95thpercentile + - .*:fifteenminuterate + - .*:fiveminuterate + - .*:durationunit + - .*:rateunit + - .*:stddev + - .*:meanrate + - .*:mean + - .*:min + + # Path present in many metrics but uninterresting + - .*:viewlockacquiretime:.* + - .*:viewreadtime:.* + - .*:cas[a-z]+latency:.* + - .*:colupdatetimedeltahistogram:.* + + # Mostly for RPC, do not scrap them + # - org:apache:cassandra:db:.* + - org:apache:cassandra:db:batchlogmanager:.* + - org:apache:cassandra:db:caches:.* + - org:apache:cassandra:db:dynamicendpointsnitch:.* + - org:apache:cassandra:db:indexcolumnfamilies:.* + - org:apache:cassandra:db:indexsummaries:.* + - org:apache:cassandra:db:indextables:.* + - org:apache:cassandra:db:nativeaccess:.* + - org:apache:cassandra:db:storageproxy:.* + + # columnfamily is an alias for Table metrics + # https://github.com/apache/cassandra/blob/8b3a60b9a7dbefeecc06bace617279612ec7092d/src/java/org/apache/cassandra/metrics/TableMetrics.java#L162 + - org:apache:cassandra:metrics:columnfamily:.* + + # Should we export metrics for system keyspaces/tables ? + - org:apache:cassandra:metrics:[^:]+:system[^:]*:.* + + # Some another uninterresting metrics + - org:apache:cassandra:metrics:indextable:.* + - org:apache:cassandra:metrics:indexcolumnfamily:.* + + - .*:waitingonfreememtablespace:.* + - .*:estimatedpartitionsizehistogram:.* + - .*:estimatedcolumncounthistogram:.* + + # Don't scrap us + - com:criteo:nosql:cassandra:exporter:.* + +maxScrapFrequencyInSec: + 15: + - java:lang:garbagecollector:.* + - java:lang:memory:.* + - java:lang:threading:.* + - org:apache:cassandra:db:compactionmanager:compactions + - org:apache:cassandra:db:compactionmanager:corecompactorthreads + - org:apache:cassandra:db:compactionmanager:corevalidationthreads + - org:apache:cassandra:db:compactionmanager:maximumcompactorthreads + - org:apache:cassandra:db:storageservice:livenodes + - org:apache:cassandra:db:storageservice:unreachablenodes + - org:apache:cassandra:metrics:cache:keycache:hits:count + - org:apache:cassandra:metrics:cache:keycache:requests:count + - org:apache:cassandra:metrics:client:connectednativeclients:value + - org:apache:cassandra:metrics:clientrequest:.*:latency:count + - org:apache:cassandra:metrics:clientrequest:read:latency:oneminuterate + - org:apache:cassandra:metrics:clientrequest:read:timeouts:count + - org:apache:cassandra:metrics:clientrequest:read:totallatency:count + - org:apache:cassandra:metrics:clientrequest:read:unavailables:count + - org:apache:cassandra:metrics:clientrequest:write:latency:oneminuterate + - org:apache:cassandra:metrics:clientrequest:write:timeouts:count + - org:apache:cassandra:metrics:clientrequest:write:totallatency:count + - org:apache:cassandra:metrics:clientrequest:write:unavailables:count + - org:apache:cassandra:metrics:compaction:bytescompacted:count + - org:apache:cassandra:metrics:compaction:pendingtasks:value + - org:apache:cassandra:metrics:compaction:totalcompactionscompleted:count + - org:apache:cassandra:metrics:storage:.* + - org:apache:cassandra:metrics:table:[^:]+:[^:]+:percentrepaired:value + - org:apache:cassandra:metrics:table:percentrepaired:value + - org:apache:cassandra:metrics:threadpools:.*:activetasks:value + - org:apache:cassandra:metrics:threadpools:.*:pendingtasks:value + - org:apache:cassandra:metrics:threadpools:.*:totalblockedtasks:count + - org:apache:cassandra:metrics:table:.*:.*:readlatency:.* + - org:apache:cassandra:metrics:table:.*:.*:writelatency:.* + - org:apache:cassandra:metrics:table:.*:.*:rangelatency:.* + - org:apache:cassandra:metrics:table:.*:.*:sstablesperreadhistogram:.* + - org:apache:cassandra:metrics:table:.*:.*:livesstablecount:value + - org:apache:cassandra:metrics:table:.*:.*:pendingcompactions:value + - org:apache:cassandra:metrics:table:.*:.*:pendingflushes:count + - org:apache:cassandra:metrics:table:.*:.*:compactionbyteswritten:count + - org:apache:cassandra:metrics:streaming:totalincomingbytes:count + - org:apache:cassandra:metrics:streaming:totaloutgoingbytes:count + - org:apache:cassandra:metrics:table:.*:.*:tombstonescannedhistogram:.* + - org:apache:cassandra:metrics:commitlog:completedtasks:value + - org:apache:cassandra:metrics:table:.*:.*:memtableswitchcount:.*" + + 60: + - org:apache:cassandra:metrics:droppedmessage:.*:oneminuterate + - org:apache:cassandra:metrics:connection:totaltimeouts:oneminuterate + - org:apache:cassandra:metrics:table:.*:.*:readlatency:oneminuterate + - org:apache:cassandra:metrics:table:.*:.*:keycachehitrate:value + - org:apache:cassandra:metrics:cache:chunkcache:oneminutehitrate:value + - org:apache:cassandra:metrics:connection:totaltimeouts:oneminuterate + - org:apache:cassandra:metrics:table:.*:.*:writelatency:oneminuterate + - org:apache:cassandra:metrics:commitlog:waitingoncommit:oneminuterate + 120: + - org:apache:cassandra:metrics:table:.*:.*:rowcachehit:count + - org:apache:cassandra:metrics:table:.*:.*:rowcachemiss:count + - org:apache:cassandra:metrics:threadpools:.*:completedtasks:value + - org:apache:cassandra:metrics:table:.*:.*:allmemtablesheapsize:value + - org:apache:cassandra:metrics:table:.*:.*:allmemtableslivedatasize:value + - org:apache:cassandra:metrics:table:$keyspace:.*:waitingonfreememtablespace:max + + # Refresh those metrics only every hour as it is costly for cassandra to retrieve them + 3600: + - .*:snapshotssize:.* + - .*:estimated.* + - .*:totaldiskspaceused:.* diff --git a/ansible/roles/cassandra-exporter/templates/cassandra_exporter.sh.j2 b/ansible/roles/cassandra-exporter/templates/cassandra_exporter.sh.j2 new file mode 100644 index 000000000..f57a253e7 --- /dev/null +++ b/ansible/roles/cassandra-exporter/templates/cassandra_exporter.sh.j2 @@ -0,0 +1,2 @@ +#!/bin/bash +java -jar {{ cassandra_exporter_dist_dir }}/cassandra.jar {{ cassandra_exporter_config_dir }}/config.yml diff --git a/ansible/roles/cassandra-exporter/templates/etc/systemd/system/prometheus-cassandra-exporter.service.j2 b/ansible/roles/cassandra-exporter/templates/etc/systemd/system/prometheus-cassandra-exporter.service.j2 new file mode 100644 index 000000000..f0aac29b6 --- /dev/null +++ b/ansible/roles/cassandra-exporter/templates/etc/systemd/system/prometheus-cassandra-exporter.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Prometheus Cassandra Exporter +After=network.target + +[Service] +Type=simple +User={{ cassandra_exporter_user }} +Group={{ cassandra_exporter_group }} +ExecStart={{ cassandra_exporter_dist_dir }}/cassandra_exporter.sh + +SyslogIdentifier=prometheus_cassandra_exporter +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/cassandra-exporter/tests/inventory b/ansible/roles/cassandra-exporter/tests/inventory new file mode 100644 index 000000000..878877b07 --- /dev/null +++ b/ansible/roles/cassandra-exporter/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/ansible/roles/cassandra-exporter/tests/test.yml b/ansible/roles/cassandra-exporter/tests/test.yml new file mode 100644 index 000000000..4c29ffaef --- /dev/null +++ b/ansible/roles/cassandra-exporter/tests/test.yml @@ -0,0 +1,6 @@ +--- +- hosts: localhost + become: true + become_method: sudo + roles: + - ansible-cassandra-exporter diff --git a/ansible/roles/cassandra-exporter/vars/main.yml b/ansible/roles/cassandra-exporter/vars/main.yml new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/ansible/roles/cassandra-exporter/vars/main.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/cassandra-restore/tasks/main.yml b/ansible/roles/cassandra-restore/tasks/main.yml index b3e2a5447..fbdaeae43 100755 --- a/ansible/roles/cassandra-restore/tasks/main.yml +++ b/ansible/roles/cassandra-restore/tasks/main.yml @@ -8,8 +8,8 @@ - name: Download to azure blob storage command: az storage blob download --name {{ cassandra_restore_gzip_file_name }} --file {{ cassandra_restore_gzip_file_path }} --container-name {{ cassandra_backup_azure_container_name }} environment: - AZURE_STORAGE_ACCOUNT: "{{ cassandra_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ cassandra_backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" async: 3600 poll: 10 @@ -47,4 +47,4 @@ become: yes service: name=cassandra state=started - \ No newline at end of file + diff --git a/ansible/roles/cassandra/defaults/main.yml b/ansible/roles/cassandra/defaults/main.yml index 63145cb92..e0a1efbfa 100644 --- a/ansible/roles/cassandra/defaults/main.yml +++ b/ansible/roles/cassandra/defaults/main.yml @@ -1,3 +1,4 @@ user_home: /home/deployer cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main' cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS' +resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra diff --git a/ansible/roles/cassandra/templates/cassandra-env.sh b/ansible/roles/cassandra/templates/cassandra-env.sh index ea781665e..77f85e560 100644 --- a/ansible/roles/cassandra/templates/cassandra-env.sh +++ b/ansible/roles/cassandra/templates/cassandra-env.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -49,42 +50,28 @@ calculate_heap_sizes() system_cpu_cores="1" fi - # set max heap size based on the following - # max(min(1/2 ram, 1024MB), min(1/4 ram, 8GB)) - # calculate 1/2 ram and cap to 1024MB - # calculate 1/4 ram and cap to 8192MB - # pick the max + # New Logic to pick the heap size + # If resource_crunch == yes then take quarter size of the Server + # else take half the size of the server and if half server Size exceeds 12GB then always take 12GB as the max heap size + half_system_memory_in_mb=`expr $system_memory_in_mb / 2` quarter_system_memory_in_mb=`expr $half_system_memory_in_mb / 2` - max_heap_size_in_mb="$half_system_memory_in_mb" - #if [ "$half_system_memory_in_mb" -gt "1024" ] - #then - # half_system_memory_in_mb="1024" - #fi - #if [ "$quarter_system_memory_in_mb" -gt "8192" ] - #then - # quarter_system_memory_in_mb="8192" - #fi - #if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ] - #then - # max_heap_size_in_mb="$half_system_memory_in_mb" - #else - # max_heap_size_in_mb="$quarter_system_memory_in_mb" - #fi - MAX_HEAP_SIZE="${half_system_memory_in_mb}M" - - # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap size) - max_sensible_yg_per_core_in_mb="100" - max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*" $system_cpu_cores` - - desired_yg_in_mb=`expr $max_heap_size_in_mb / 4` - - if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ] + + resource_crunch="{{resource_crunch}}" + if [ $resource_crunch = "yes" ]; then - HEAP_NEWSIZE="${max_sensible_yg_in_mb}M" - else - HEAP_NEWSIZE="${desired_yg_in_mb}M" + MAX_HEAP_SIZE="${quarter_system_memory_in_mb}M" fi + + if [ $resource_crunch != "yes" -a $half_system_memory_in_mb > 12288 ]; + then + MAX_HEAP_SIZE="12288M" + elif [ "$resource_crunch" != "yes" ] + then + MAX_HEAP_SIZE="${half_system_memory_in_mb}M" + fi + + HEAP_NEWSIZE=$MAX_HEAP_SIZE } # Determine the sort of JVM we'll be running on. @@ -295,3 +282,4 @@ JVM_OPTS="$JVM_OPTS -Djava.library.path=$CASSANDRA_HOME/lib/sigar-bin" JVM_OPTS="$JVM_OPTS $MX4J_ADDRESS" JVM_OPTS="$JVM_OPTS $MX4J_PORT" JVM_OPTS="$JVM_OPTS $JVM_EXTRA_OPTS" + diff --git a/ansible/roles/cassandra/templates/cassandra.yaml.j2 b/ansible/roles/cassandra/templates/cassandra.yaml.j2 index 0192f9324..adb2d867e 100755 --- a/ansible/roles/cassandra/templates/cassandra.yaml.j2 +++ b/ansible/roles/cassandra/templates/cassandra.yaml.j2 @@ -83,7 +83,7 @@ compaction_throughput_mb_per_sec: 16 sstable_preemptive_open_interval_in_mb: 50 read_request_timeout_in_ms: 5000 range_request_timeout_in_ms: 10000 -write_request_timeout_in_ms: 2000 +write_request_timeout_in_ms: 5000 counter_write_request_timeout_in_ms: 5000 cas_contention_timeout_in_ms: 1000 truncate_request_timeout_in_ms: 60000 diff --git a/ansible/roles/cdn-upload/defaults/main.yml b/ansible/roles/cdn-upload/defaults/main.yml deleted file mode 100644 index 76dd374d1..000000000 --- a/ansible/roles/cdn-upload/defaults/main.yml +++ /dev/null @@ -1,3 +0,0 @@ -player_cdn_container: -player_cdn_account: -assets: diff --git a/ansible/roles/cdn-upload/meta/main.yml b/ansible/roles/cdn-upload/meta/main.yml deleted file mode 100644 index 23b18a800..000000000 --- a/ansible/roles/cdn-upload/meta/main.yml +++ /dev/null @@ -1,2 +0,0 @@ -dependencies: - - azure-cli \ No newline at end of file diff --git a/ansible/roles/cdn-upload/tasks/main.yml b/ansible/roles/cdn-upload/tasks/main.yml deleted file mode 100755 index c4fe89b33..000000000 --- a/ansible/roles/cdn-upload/tasks/main.yml +++ /dev/null @@ -1,16 +0,0 @@ -- name: Ensure azure blob storage container exists - command: az storage container create --name {{player_cdn_container}} - ignore_errors: true - environment: - AZURE_STORAGE_ACCOUNT: "{{player_cdn_account}}" - AZURE_STORAGE_KEY: "{{player_cdn_account_key}}" - -# Upload the assets created by the job to azure -- name: Upload to azure blob storage - command: az storage blob upload-batch -d {{player_cdn_container}} -s {{assets}} - environment: - AZURE_STORAGE_ACCOUNT: "{{player_cdn_account}}" - AZURE_STORAGE_KEY: "{{player_cdn_account_key}}" - async: 60 - poll: 10 - diff --git a/ansible/roles/cert-file-upload/defaults/main.yml b/ansible/roles/cert-file-upload/defaults/main.yml new file mode 100644 index 000000000..b74bef14c --- /dev/null +++ b/ansible/roles/cert-file-upload/defaults/main.yml @@ -0,0 +1,2 @@ +file_name: "certUploadedfile" +cert_template_artifacts_container_name: "e-credentials" diff --git a/ansible/roles/cert-file-upload/tasks/main.yml b/ansible/roles/cert-file-upload/tasks/main.yml new file mode 100644 index 000000000..9fcaf6a67 --- /dev/null +++ b/ansible/roles/cert-file-upload/tasks/main.yml @@ -0,0 +1,7 @@ +- name: rename the file to the correct name as required in blobstore + command: "mv /tmp/certUploadedfile /tmp/{{file_name}}" + +- name: Upload to azure blob storage + command: "az storage blob upload -c {{ cert_template_artifacts_container_name }} --name {{ file_name }} -f /tmp/{{file_name}}" + async: 3600 + poll: 10 diff --git a/ansible/roles/cert-templates/defaults/main.yml b/ansible/roles/cert-templates/defaults/main.yml new file mode 100644 index 000000000..c8710dd9d --- /dev/null +++ b/ansible/roles/cert-templates/defaults/main.yml @@ -0,0 +1,4 @@ +certs_badge_upload_retry_count: 3 +certs_badge_criteria: "" +certs_badge_batch_id: "" +certs_badge_key_id: "" diff --git a/ansible/roles/cert-templates/tasks/main.yml b/ansible/roles/cert-templates/tasks/main.yml index ae1f5160a..b41aaef0a 100644 --- a/ansible/roles/cert-templates/tasks/main.yml +++ b/ansible/roles/cert-templates/tasks/main.yml @@ -34,16 +34,16 @@ - name: Ensure azure blob storage container exists command: az storage container create --name {{cert_service_container_name}} --public-access off environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_azure_storage_account}}" - AZURE_STORAGE_KEY: "{{sunbird_azure_storage_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}" - name: Upload to azure blob storage command: az storage blob upload-batch --destination {{cert_service_container_name}} --source "out" args: chdir: "{{cert_location}}/cert-templates/certUtilScripts/" environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_azure_storage_account}}" - AZURE_STORAGE_KEY: "{{sunbird_azure_storage_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}" async: 60 poll: 10 diff --git a/ansible/roles/cert-templates/templates/setVars.sh.j2 b/ansible/roles/cert-templates/templates/setVars.sh.j2 index 6ea40c8a4..b4e9a8e1d 100644 --- a/ansible/roles/cert-templates/templates/setVars.sh.j2 +++ b/ansible/roles/cert-templates/templates/setVars.sh.j2 @@ -26,7 +26,7 @@ export BATCH_ID="{{certs_badge_batch_id}}" export CONTAINER_NAME="{{cert_service_container_name}}" export CLOUD_STORAGE_TYPE="{{cert_service_cloud_storage_type}}" export CLOUD_UPLOAD_RETRY_COUNT="{{certs_badge_upload_retry_count}}" -export AZURE_STORAGE_SECRET="{{sunbird_azure_storage_key}}" -export AZURE_STORAGE_KEY="{{sunbird_azure_storage_account}}" +export AZURE_STORAGE_SECRET="{{sunbird_private_storage_account_key}}" +export AZURE_STORAGE_KEY="{{sunbird_private_storage_account_name}}" ################### END - configuration ############################# diff --git a/ansible/roles/desktop-deploy/defaults/main.yml b/ansible/roles/desktop-deploy/defaults/main.yml new file mode 100644 index 000000000..d71509fd0 --- /dev/null +++ b/ansible/roles/desktop-deploy/defaults/main.yml @@ -0,0 +1,2 @@ +--- +time: "YEAR-MONTH-DATE-HOUR-MINUTE-SECOND-INSTALLERTYPE" diff --git a/ansible/roles/desktop-deploy/tasks/main.yml b/ansible/roles/desktop-deploy/tasks/main.yml new file mode 100644 index 000000000..0d301fbed --- /dev/null +++ b/ansible/roles/desktop-deploy/tasks/main.yml @@ -0,0 +1,76 @@ +--- +- name: Copy the desktop assests folder from inside the private repo location to root location. + shell: "cp -r {{offline_repo_location}}/private/desktop-assets {{offline_repo_location}}/" + +- name: get the date and time for the artifact + set_fact: + time: "{{ lookup('pipe', 'date +\"%Y-%b-%d\"') }}-{{offline_installer_type}}" + +- name: create a directory to store artifacts + file: + path: "{{offline_repo_location}}/desktop_uploader_assets/{{time}}" + state: directory + recurse: yes + +- name: Get the environment name for the artifact name + shell: "cat {{offline_repo_location}}/desktop-assets/{{offline_target_env}}/appConfig.json | jq -r '.APP_NAME'" + register: env_name + +- name: Display the environment name of the installer + debug: + msg: "{{env_name.stdout}}" + +- name: Create a variable to inject environment name to upload to azure blob + set_fact: + environment_name: "{{ env_name.stdout }}" + +- name: Get the version from the package.json file + shell: "cat {{offline_repo_location}}/app_dist/package.json | jq -r '.version'" + register: version + +- name: Display the version number of the installer + debug: + msg: "{{version.stdout}}" + +- name: Create a variable to inject version in the template + set_fact: + installer_version: "{{ version.stdout }}" + +- name: copy the installer script file and build script for building offline installer + template: + src: "{{item}}.j2" + dest: "{{offline_repo_location}}/{{item}}" + mode: '0755' + with_items: + - build.sh + - envfile + - setupOfflineInstaller.sh + +- name: run the installer script + shell: "bash -x {{offline_repo_location}}/build.sh" + +- name: Ensure azure blob storage container exists + command: az storage container create --name {{offline_installer_container_name}} --public-access blob + environment: + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" + +- name: Upload to azure blob storage + command: az storage blob upload-batch --destination {{offline_installer_container_name}} --source "desktop_uploader_assets" + args: + chdir: "{{offline_repo_location}}/" + environment: + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" + async: 60 + poll: 10 + +- name: Upload to latest.json file to blob + command: az storage blob upload-batch --destination "{{offline_installer_container_name}}/latest" --source "{{offline_repo_location}}/desktop_uploader_assets/{{time}}/" + args: + chdir: "{{offline_repo_location}}/" + environment: + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" + async: 60 + poll: 10 diff --git a/ansible/roles/desktop-deploy/templates/build.sh.j2 b/ansible/roles/desktop-deploy/templates/build.sh.j2 new file mode 100644 index 000000000..2f431915e --- /dev/null +++ b/ansible/roles/desktop-deploy/templates/build.sh.j2 @@ -0,0 +1,28 @@ +#!/bin/bash +set -eo pipefail + +cd {{offline_repo_location}}/ + +# Run the docker image and run the OS Specific build along with environment specific build +docker run -d --env-file envfile --env ELECTRON_CACHE="/root/.cache/electron" --env ELECTRON_BUILDER_CACHE="/root/.cache/electron-builder" --name offline_deploy -w /project electronuserland/builder:wine sleep infinity +docker cp . offline_deploy:/project/ +docker exec offline_deploy bash -x /project/setupOfflineInstaller.sh + +# Copy the built artifacts +if [ "{{offline_installer_type}}" == "windows32bit" ]; +then + docker cp 'offline_deploy:/project/app_dist/dist/{{installer_version}}/win/ia32/{{environment_name}} Setup {{installer_version}}.exe' '{{offline_repo_location}}/desktop_uploader_assets/{{time}}/{{environment_name}}_{{installer_version}}_windows32bit.exe' + + elif [ "{{offline_installer_type}}" == "windows64bit" ]; + then + docker cp 'offline_deploy:/project/app_dist/dist/{{installer_version}}/win/x64/{{environment_name}} Setup {{installer_version}}.exe' '{{offline_repo_location}}/desktop_uploader_assets/{{time}}/{{environment_name}}_{{installer_version}}_windows64bit.exe' + + elif [ "{{offline_installer_type}}" == "linux64bit" ]; + then + docker cp 'offline_deploy:/project/app_dist/dist/{{installer_version}}/linux/x64/{{environment_name}}_{{installer_version}}_amd64.deb' '{{offline_repo_location}}/desktop_uploader_assets/{{time}}/{{environment_name}}_{{installer_version}}_linux64bit.deb' +fi + +# Generate the latest.json file +echo "{\"version\":\"{{installer_version}}\",\"windows\":{\"32bit\":\"{{environment_name}}_{{installer_version}}_windows32bit.exe\",\"64bit\":\"{{environment_name}}_{{installer_version}}_windows64bit.exe\"},\"linux\":{\"64bit\":\"{{environment_name}}_{{installer_version}}_linux64bit.deb\"}}" | jq '.' | tee -a '{{offline_repo_location}}/desktop_uploader_assets/{{time}}/latest.json' + +echo "Build the installer succesfully" diff --git a/ansible/roles/desktop-deploy/templates/envfile.j2 b/ansible/roles/desktop-deploy/templates/envfile.j2 new file mode 100644 index 000000000..576b57e74 --- /dev/null +++ b/ansible/roles/desktop-deploy/templates/envfile.j2 @@ -0,0 +1,3 @@ +offline_desktop_assets_token={{offline_desktop_assets_token}} +offline_target_env={{offline_target_env}} +offline_app_base_url_token={{offline_app_base_url_token}} diff --git a/ansible/roles/desktop-deploy/templates/setupOfflineInstaller.sh.j2 b/ansible/roles/desktop-deploy/templates/setupOfflineInstaller.sh.j2 new file mode 100644 index 000000000..f569c990e --- /dev/null +++ b/ansible/roles/desktop-deploy/templates/setupOfflineInstaller.sh.j2 @@ -0,0 +1,22 @@ +#!/bin/bash + +cd /project/app_dist + +if [ "{{offline_installer_type}}" != "windows32bit" ]; +then + #Build the offline installer + yarn install + node scripts/metadata.js + rm -rf scripts +fi + +if [ "{{offline_installer_type}}" == "windows32bit" ]; +then + npm run dist-win32 +elif [ "{{offline_installer_type}}" == "windows64bit" ]; +then + npm run dist-win64 +elif [ "{{offline_installer_type}}" == "linux64bit" ]; +then + npm run dist-linux +fi diff --git a/ansible/roles/es-azure-restore/defaults/main.yml b/ansible/roles/es-azure-restore/defaults/main.yml index b5de80080..2f70c0d69 100644 --- a/ansible/roles/es-azure-restore/defaults/main.yml +++ b/ansible/roles/es-azure-restore/defaults/main.yml @@ -1,4 +1,6 @@ -# Override these values +es_restore_host: localhost +snapshot_base_path: application + snapshot_create_request_body: { type: azure, settings: { @@ -7,5 +9,4 @@ snapshot_create_request_body: { } } -es_restore_host: "{{groups['es'][0]}}" -snapshot_base_path: application +#snapshot_base_path: folderNameInTheAzureContainer diff --git a/ansible/roles/es-azure-restore/tasks/main.yml b/ansible/roles/es-azure-restore/tasks/main.yml index 0c46e6185..f45a97587 100644 --- a/ansible/roles/es-azure-restore/tasks/main.yml +++ b/ansible/roles/es-azure-restore/tasks/main.yml @@ -7,9 +7,6 @@ headers: Content-Type: "application/json" -- debug: - var: snapshot_base_path - - name: Restore ES from Azure backup uri: url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore" diff --git a/ansible/roles/es-mapping/files/indices/userfeed.json b/ansible/roles/es-mapping/files/indices/userfeed.json new file mode 100644 index 000000000..dd7eaeade --- /dev/null +++ b/ansible/roles/es-mapping/files/indices/userfeed.json @@ -0,0 +1,212 @@ +{ + "settings": { + "index": { + "number_of_shards": "5", + "number_of_replicas": "1", + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "ngram", + "max_gram": "20" + } + }, + "analyzer": { + "cs_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "type": "custom", + "tokenizer": "keyword" + }, + "cs_search_analyzer": { + "filter": [ + "lowercase", + "standard" + ], + "type": "custom", + "tokenizer": "standard" + } + } + } + } + }, + "mappings": { + "_doc": { + "dynamic":false, + "properties": { + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "category": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "data": { + "type": "object" + }, + "createdBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "priority": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "createdOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "expireOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "updatedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "updatedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "userId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/es-mapping/files/mappings/user-mapping.json b/ansible/roles/es-mapping/files/mappings/user-mapping.json index 56c961e55..99622c94e 100644 --- a/ansible/roles/es-mapping/files/mappings/user-mapping.json +++ b/ansible/roles/es-mapping/files/mappings/user-mapping.json @@ -1,6 +1,22 @@ { "dynamic": false, "properties": { + "managedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, "activeStatus": { "type": "text", "fields": { @@ -3537,4 +3553,4 @@ } } } -} \ No newline at end of file +} diff --git a/ansible/roles/es-mapping/files/mappings/userfeed-mapping.json b/ansible/roles/es-mapping/files/mappings/userfeed-mapping.json new file mode 100644 index 000000000..deba277e5 --- /dev/null +++ b/ansible/roles/es-mapping/files/mappings/userfeed-mapping.json @@ -0,0 +1,163 @@ +{ + "dynamic":false, + "properties": { + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "category": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "data": { + "type": "object" + }, + "createdBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "priority": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "createdOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "expireOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "updatedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "updatedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "userId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + } + } +} \ No newline at end of file diff --git a/ansible/roles/es6/defaults/main.yml b/ansible/roles/es6/defaults/main.yml index f0ce74ac3..b2402c73a 100644 --- a/ansible/roles/es6/defaults/main.yml +++ b/ansible/roles/es6/defaults/main.yml @@ -41,3 +41,4 @@ conf_dir: '' data_dirs: '' # JVM custom parameters es_jvm_custom_parameters: '' +es_heap_size: "2g" diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml index bbb2903f0..7bf1a214a 100644 --- a/ansible/roles/es6/tasks/main.yml +++ b/ansible/roles/es6/tasks/main.yml @@ -79,3 +79,15 @@ when: es_templates tags: - templates + +# Add ES Plugin credentials for backups +- name: Add ES Backup variables to elasticsearch to have backups enabled + lineinfile: + path: '{{conf_dir}}/elasticsearch.yml' + line: 'cloud.azure.storage.default.{{item.key}}: {{item.value}}' + regexp: "cloud.azure.storage.default.{{ item.key }}.*" + with_items: + - {key: 'account', value: '{{ sunbird_management_storage_account_name }}'} + - {key: 'key', value: '{{ sunbird_management_storage_account_key }}'} + when: sunbird_management_storage_account_name is defined and sunbird_management_storage_account_key is defined + notify: restart elasticsearch diff --git a/ansible/roles/es6/templates/elasticsearch.yml.j2 b/ansible/roles/es6/templates/elasticsearch.yml.j2 index 3cdbe5ab7..030cc6e1a 100644 --- a/ansible/roles/es6/templates/elasticsearch.yml.j2 +++ b/ansible/roles/es6/templates/elasticsearch.yml.j2 @@ -7,6 +7,12 @@ cluster.name: elasticsearch {% endif %} +{% if (groups['es']|length) <= 2 %} +discovery.zen.minimum_master_nodes: 1 +{% else %} +discovery.zen.minimum_master_nodes: "{{ ((groups['es']|length) / 2 +1) | round(0, 'floor') | int}}" +{% endif %} + {% if es_config['node.name'] is not defined %} node.name: {{inventory_hostname}}-{{es_instance_name}} {% endif %} @@ -50,8 +56,3 @@ network.host: 0.0.0.0 {% if es_remote_reindex is defined %} reindex.remote.whitelist: {{es_remote_host}}:9200 {% endif %} - -{% if backup_azure_storage_account_name is defined and backup_azure_storage_access_key is defined %} -cloud.azure.storage.default.account: {{ backup_azure_storage_account_name }} -cloud.azure.storage.default.key: {{ backup_azure_storage_access_key }} -{% endif %} diff --git a/ansible/roles/esUpgrade/defaults/main.yml b/ansible/roles/esUpgrade/defaults/main.yml index 5728f2916..ded08a753 100755 --- a/ansible/roles/esUpgrade/defaults/main.yml +++ b/ansible/roles/esUpgrade/defaults/main.yml @@ -23,5 +23,5 @@ es_plugins: es_plugins_reinstall: true # Override these values in group_vars -backup_storage_name: backups -backup_storage_key: '' +sunbird_management_storage_account_name: backups +sunbird_management_storage_account_key: '' diff --git a/ansible/roles/esUpgrade/meta/main.yml b/ansible/roles/esUpgrade/meta/main.yml index 56c9566c7..f1420c7b3 100755 --- a/ansible/roles/esUpgrade/meta/main.yml +++ b/ansible/roles/esUpgrade/meta/main.yml @@ -17,8 +17,8 @@ dependencies: discovery.zen.minimum_master_nodes: "{{groups['es-upgrade']| length | int}}", bootstrap.memory_lock: true, cloud.azure.storage.my_account: { - account: "{{ backup_storage_name }}", - key: "{{ backup_storage_key }}" + account: "{{ sunbird_management_storage_account_name }}", + key: "{{ sunbird_management_storage_account_key }}" } }, es_plugins: [ diff --git a/ansible/roles/firebase_deploy/defaults/main.yml b/ansible/roles/firebase_deploy/defaults/main.yml new file mode 100644 index 000000000..49e165755 --- /dev/null +++ b/ansible/roles/firebase_deploy/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for mobile_upload \ No newline at end of file diff --git a/ansible/roles/firebase_deploy/tasks/main.yml b/ansible/roles/firebase_deploy/tasks/main.yml new file mode 100644 index 000000000..0403ff7da --- /dev/null +++ b/ansible/roles/firebase_deploy/tasks/main.yml @@ -0,0 +1,17 @@ +--- +# tasks file for firebase_deploy + - name: Templating Firebase build upload script + template: + src: "deployToFirebase.sh" + dest: "./deployToFirebase.sh" + - name: Templating GDrive build upload script + template: + src: "uploadToGdrive.sh" + dest: "./uploadToGdrive.sh" + - name: Uploading build to {{ env_name }} firebase + shell: find ../ -maxdepth 1 -iregex ".*{{env_name}}.apk" -exec bash deployToFirebase.sh {} \; + when: env_name!='production' + - name: Uploading build to {{ env_name }} Gdrive + shell: find ../ -maxdepth 1 -iregex ".*[0-9].apk" -exec bash uploadToGdrive.sh -v -r {} \; + when: env_name=='production' + diff --git a/ansible/roles/firebase_deploy/templates/deployToFirebase.sh b/ansible/roles/firebase_deploy/templates/deployToFirebase.sh new file mode 100755 index 000000000..879ef039e --- /dev/null +++ b/ansible/roles/firebase_deploy/templates/deployToFirebase.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Upload build to Firebase +# Usage: upload.sh + +#!/bin/bashset -e +echo "Build to upload "${1} +firebase appdistribution:distribute ${1} \ + --token {{mobile_firebse_app_distribution_token}} \ + --app {{mobile_firebse_app_distribution_id}} \ + --groups {{mobile_firebse_app_distribution_group}} diff --git a/ansible/roles/firebase_deploy/templates/uploadToGdrive.sh b/ansible/roles/firebase_deploy/templates/uploadToGdrive.sh new file mode 100755 index 000000000..9dc70a7b5 --- /dev/null +++ b/ansible/roles/firebase_deploy/templates/uploadToGdrive.sh @@ -0,0 +1,343 @@ +#!/bin/bash + +# Upload a file to Google Drive +# Usage: upload.sh + +#!/bin/bashset -e +function usage(){ + + echo -e "\nThe script can be used to upload file/directory to google drive." + echo -e "\nUsage:\n $0 [options..] \n" + echo -e "Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive. \n" + echo -e "File name argument is optional if create directory option is used. \n" + echo -e "Options:\n" + echo -e "-C | --create-dir - option to create directory. Will provide folder id." + echo -e "-r | --root-dir - google folder id to which the file/directory to upload." + echo -e "-v | --verbose - Display detailed message." + echo -e "-z | --config - Override default config file with custom config file." + echo -e "-h | --help - Display usage instructions.\n" + exit 0; +} + +file=${3} +#Configuration variables +ROOT_FOLDER="" +CLIENT_ID={{mobile_gdrive_client_id}} +CLIENT_SECRET={{mobile_gdrive_client_secret}} +API_REFRESH_TOKEN={{mobile_gdrive_refresh_token}} +FOLDER_ID={{mobile_release_gdrive_folder_id}} +FILE_NAME=${3} +SCOPE=${SCOPE:-"https://docs.google.com/feeds"} + +echo "FILE"${3} +#Internal variable +ACCESS_TOKEN="" +FILE="" +FOLDERNAME="" +curl_args="" + +DIR="$( cd "$( dirname "$( readlink "${BASH_SOURCE[0]}" )" )" && pwd )" + +if [ -e $HOME/.googledrive.conf ] +then + . $HOME/.googledrive.conf +fi + +PROGNAME=${0##*/} +SHORTOPTS="vhr:C:z:" +LONGOPTS="verbose,help,create-dir:,root-dir:,config:" + +set -o errexit -o noclobber -o pipefail #-o nounset +OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name $PROGNAME -- "$@" ) + +# script to parse the input arguments +#if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi + +eval set -- "$OPTS" + +VERBOSE=false +HELP=false +CONFIG="" +ROOTDIR="" + +while true; do + case "$1" in + -v | --verbose ) VERBOSE=true;curl_args="--progress"; shift ;; + -h | --help ) usage; shift ;; + -C | --create-dir ) FOLDERNAME="$2"; shift 2 ;; + -r | --root-dir ) ROOTDIR="$2";ROOT_FOLDER="$2"; shift 2 ;; + -z | --config ) CONFIG="$2"; shift 2 ;; + -- ) shift; break ;; + * ) break ;; + esac +done + +if [ ! -z "$CONFIG" ] + then + if [ -e "$CONFIG" ] + then + . $CONFIG + fi + if [ ! -z "$ROOTDIR" ] + then + ROOT_FOLDER="$ROOTDIR" + fi + +fi + +if [ ! -z "$1" ] +then + FILE=$1 +fi + +if [ ! -z "$2" ] && [ -z "$FOLDERNAME" ] +then + FOLDERNAME=$2 +fi + + +function log() { + + if [ "$VERBOSE" = true ]; then + echo -e "${1}" + + fi +} + +# Method to extract data from json response +function jsonValue() { +KEY=$1 +num=$2 +awk -F"[,:}][^://]" '{for(i=1;i<=NF;i++){if($i~/\042'$KEY'\042/){print $(i+1)}}}' | tr -d '"' | sed -n ${num}p | sed -e 's/[}]*$//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/[,]*$//' +} +# Method to upload files to google drive. Requires 3 arguments file path, google folder id and access token. +function uploadFile(){ + FILE="$1" + FOLDER_ID="$2" + ACCESS_TOKEN="$3" + SLUG=`basename "$FILE"` + FILENAME="${SLUG%.*}" + EXTENSION="${SLUG##*.}" + if [ "$FILENAME" == "$EXTENSION" -o ! "$(command -v mimetype)" ] + then + MIME_TYPE=`file --brief --mime-type "$FILE"` + else + MIME_TYPE=`mimetype --output-format %m "$FILE"` + + fi + + + FILESIZE=$(stat -c%s "$FILE") + + # JSON post data to specify the file name and folder under while the file to be created + postData="{\"mimeType\": \"$MIME_TYPE\",\"title\": \"$SLUG\",\"parents\": [{\"id\": \"${FOLDER_ID}\"}]}" + postDataSize=$(echo $postData | wc -c) + # Curl command to initiate resumable upload session and grab the location URL + log "Generating upload link for file $FILE ..." + uploadlink=`/usr/bin/curl \ + --silent \ + -X POST \ + -H "Host: www.googleapis.com" \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -H "X-Upload-Content-Type: $MIME_TYPE" \ + -H "X-Upload-Content-Length: $FILESIZE" \ + -d "$postData" \ + "https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ + --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n'` + + # Curl command to push the file to google drive. + # If the file size is large then the content can be split to chunks and uploaded. + # In that case content range needs to be specified. + log "Uploading file $FILE to google drive..." + curl \ + --silent \ + -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: $MIME_TYPE" \ + -H "Content-Length: $FILESIZE" \ + -H "Slug: $SLUG" \ + --upload-file "$FILE" \ + --output /dev/null \ + "$uploadlink" \ + $curl_args +} +if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ] + then + echo "CLIENT_ID"${CLIENT_ID} + echo "CLIENT_SECRET"${CLIENT_SECRET} + echo "API_REFRESH_TOKEN"${API_REFRESH_TOKEN} + echo "FILE"${FILE_NAME} + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$API_REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` + echo "ACCESS_TOKEN"${ACCESS_TOKEN} + uploadFile "${FILE_NAME}" "$FOLDER_ID" "$ACCESS_TOKEN" + # usage +fi + + +# Method to create directory in google drive. Requires 3 arguments foldername,root directory id and access token. +function createDirectory(){ + DIRNAME="$1" + ROOTDIR="$2" + ACCESS_TOKEN="$3" + FOLDER_ID="" + QUERY="mimeType='application/vnd.google-apps.folder' and title='$DIRNAME'" + QUERY=$(echo $QUERY | sed -f ${DIR}/url_escape.sed) + + SEARCH_RESPONSE=`/usr/bin/curl \ + --silent \ + -XGET \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "https://www.googleapis.com/drive/v2/files/${ROOTDIR}/children?orderBy=title&q=${QUERY}&fields=items%2Fid"` + + FOLDER_ID=`echo $SEARCH_RESPONSE | jsonValue id` + + + if [ -z "$FOLDER_ID" ] + then + CREATE_FOLDER_POST_DATA="{\"mimeType\": \"application/vnd.google-apps.folder\",\"title\": \"$DIRNAME\",\"parents\": [{\"id\": \"$ROOTDIR\"}]}" + CREATE_FOLDER_RESPONSE=`/usr/bin/curl \ + --silent \ + -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -d "$CREATE_FOLDER_POST_DATA" \ + "https://www.googleapis.com/drive/v2/files?fields=id"` + FOLDER_ID=`echo $CREATE_FOLDER_RESPONSE | jsonValue id` + + fi +} + +# Method to upload files to google drive. Requires 3 arguments file path, google folder id and access token. +# function uploadFile(){ +# FILE="$1" +# FOLDER_ID="$2" +# ACCESS_TOKEN="$3" +# SLUG=`basename "$FILE"` +# FILENAME="${SLUG%.*}" +# EXTENSION="${SLUG##*.}" +# if [ "$FILENAME" == "$EXTENSION" -o ! "$(command -v mimetype)" ] +# then +# MIME_TYPE=`file --brief --mime-type "$FILE"` +# else +# MIME_TYPE=`mimetype --output-format %m "$FILE"` +# +# fi +# +# +# FILESIZE=$(stat -c%s "$FILE") +# +# # JSON post data to specify the file name and folder under while the file to be created +# postData="{\"mimeType\": \"$MIME_TYPE\",\"title\": \"$SLUG\",\"parents\": [{\"id\": \"${FOLDER_ID}\"}]}" +# postDataSize=$(echo $postData | wc -c) +# # Curl command to initiate resumable upload session and grab the location URL +# log "Generating upload link for file $FILE ..." +# uploadlink=`/usr/bin/curl \ +# --silent \ +# -X POST \ +# -H "Host: www.googleapis.com" \ +# -H "Authorization: Bearer ${ACCESS_TOKEN}" \ +# -H "Content-Type: application/json; charset=UTF-8" \ +# -H "X-Upload-Content-Type: $MIME_TYPE" \ +# -H "X-Upload-Content-Length: $FILESIZE" \ +# -d "$postData" \ +# "https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ +# --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n'` +# +# # Curl command to push the file to google drive. +# # If the file size is large then the content can be split to chunks and uploaded. +# # In that case content range needs to be specified. +# log "Uploading file $FILE to google drive..." +# curl \ +# --silent \ +# -X PUT \ +# -H "Authorization: Bearer ${ACCESS_TOKEN}" \ +# -H "Content-Type: $MIME_TYPE" \ +# -H "Content-Length: $FILESIZE" \ +# -H "Slug: $SLUG" \ +# --upload-file "$FILE" \ +# --output /dev/null \ +# "$uploadlink" \ +# $curl_args +# } + +old_umask=`umask` +umask 0077 + +if [ -z "$ROOT_FOLDER" ] +then + read -p "Root Folder ID (Default: root): " ROOT_FOLDER + if [ -z "$ROOT_FOLDER" ] || [ `echo $ROOT_FOLDER | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ] + then + ROOT_FOLDER="root" + echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf + else + if expr "$ROOT_FOLDER" : '^[A-Za-z0-9_-]\{28\}$' > /dev/null + then + echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf + else + echo "Invalid root folder id" + exit -1 + fi + fi +fi + +if [ -z "$CLIENT_ID" ] +then + read -p "Client ID: " CLIENT_ID + echo "CLIENT_ID=$CLIENT_ID" >> $HOME/.googledrive.conf +fi + +if [ -z "$CLIENT_SECRET" ] +then + read -p "Client Secret: " CLIENT_SECRET + echo "CLIENT_SECRET=$CLIENT_SECRET" >> $HOME/.googledrive.conf +fi + +if [ -z "$REFRESH_TOKEN" ] +then + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/device/code" --data "client_id=$CLIENT_ID&scope=$SCOPE"` + DEVICE_CODE=`echo "$RESPONSE" | jsonValue "device_code"` + USER_CODE=`echo "$RESPONSE" | jsonValue "user_code"` + URL=`echo "$RESPONSE" | jsonValue "verification_url"` + + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$API_REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo "$RESPONSE" | jsonValue access_token` +fi + +if [ -z "$ACCESS_TOKEN" ] + then + # Access token generation + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` +fi + +# Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder +if [ -z "$FOLDERNAME" ] || [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] + then + if [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] + then + ROOT_FOLDER="root" + fi + FOLDER_ID=$ROOT_FOLDER +else + FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` + +fi + log "Folder ID for folder name $FOLDERNAME : $FOLDER_ID" + +# Check whether the given file argument is valid and check whether the argument is file or directory. +# based on the type, if the argument is directory do a recursive upload. +if [ ! -z "$FILE" ]; then + if [ -f "$FILE" ];then + uploadFile "$FILE" "$FOLDER_ID" "$ACCESS_TOKEN" + elif [ -d "$FILE" ];then + FOLDERNAME=$(basename $FILE) + FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` + for file in $(find "$FILE" -type f); + do + uploadFile "$file" "$FOLDER_ID" "$ACCESS_TOKEN" + done + fi +fi diff --git a/ansible/roles/firebase_deploy/templates/url_escape.sed b/ansible/roles/firebase_deploy/templates/url_escape.sed new file mode 100644 index 000000000..eac400265 --- /dev/null +++ b/ansible/roles/firebase_deploy/templates/url_escape.sed @@ -0,0 +1,25 @@ +# sed url escaping +s:%:%25:g +s: :%20:g +s:<:%3C:g +s:>:%3E:g +s:#:%23:g +s:{:%7B:g +s:}:%7D:g +s:|:%7C:g +s:\\:%5C:g +s:\^:%5E:g +s:~:%7E:g +s:\[:%5B:g +s:\]:%5D:g +s:`:%60:g +s:;:%3B:g +s:/:%2F:g +s:?:%3F:g +s^:^%3A^g +s:@:%40:g +s:=:%3D:g +s:&:%26:g +s:\$:%24:g +s:\!:%21:g +s:\*:%2A:g diff --git a/ansible/roles/grafana-backup/defaults/main.yml b/ansible/roles/grafana-backup/defaults/main.yml index 8e4849c85..b32dea6dd 100644 --- a/ansible/roles/grafana-backup/defaults/main.yml +++ b/ansible/roles/grafana-backup/defaults/main.yml @@ -2,6 +2,6 @@ grafana_backup_dir: /tmp/grafana-backup grafana_data_dir: /var/dockerdata/grafana/grafana.db # Set these vars per environment as show in example below # Override these values in group_vars -backup_azure_storage_account_name: -backup_azure_storage_access_key: '' +sunbird_management_storage_account_name: +sunbird_management_storage_account_key: '' grafana_backup_azure_container_name: grafana-backup diff --git a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/keycloak/keycloak_main.py b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/keycloak/keycloak_main.py index 4dda7116b..4705499cb 100644 --- a/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/keycloak/keycloak_main.py +++ b/ansible/roles/keycloak-deploy/files/python-keycloak-0.12.0/keycloak/keycloak_main.py @@ -84,7 +84,6 @@ def update_user_roles(config): except Exception as e: if "409" in str(e): print "Skipping error: " + str(e) - pass - except Exception as e: - raise + else: + raise diff --git a/ansible/roles/keycloak-deploy/tasks/deploy.yml b/ansible/roles/keycloak-deploy/tasks/deploy.yml index 019e65a9d..a796e2ad1 100644 --- a/ansible/roles/keycloak-deploy/tasks/deploy.yml +++ b/ansible/roles/keycloak-deploy/tasks/deploy.yml @@ -1,3 +1,12 @@ +- name: Get the list of all the services running in our OS + become: yes + service_facts: + +- name: Stop the monit to ensure that it doesn't start keycloak + service: name=monit state=stopped + become: yes + when: ansible_facts.services.monit is defined + - name: Status service become: true command: /etc/init.d/keycloak status @@ -39,33 +48,24 @@ group: "{{ wildfly_group }}" force: true -- name: Paste keycloak package on server - copy: - src: "{{ artifact_path }}" - dest: "/opt/" +- name: Creating keycloak directory + become: true + file: + path: "{{ keycloak_home }}" + state: directory owner: "{{ wildfly_user }}" group: "{{ wildfly_group }}" - force: true - name: Extarct the folder unarchive: - src: "/opt/{{ artifact }}" - dest: /opt/ - copy: false + src: "{{ artifact_path }}" + dest: "{{ keycloak_home }}" owner: "{{ wildfly_user }}" group: "{{ wildfly_group }}" -- name: Move the folder name - command: mv sunbird_auth_{{ sunbird_auth_version }} keycloak - args: - chdir: /opt - -- name: Remove the opt/{{ artifact }} - become: true - file: - path: "/opt/{{ artifact }}" - state: absent - force: true +- name: Creating keycloak directory in proper structure + become: yes + shell: "cd {{ keycloak_home }} ; mv sunbird_auth_1.0v/* ./ ; rm -rf sunbird_auth_1.0v; chown -R keycloak:keycloak {{ keycloak_home }}" - name: Create required directories file: @@ -109,11 +109,19 @@ mode: 0755 force: true -- name: Keycloak configuration XML file +- name: Keycloak configuration XML file - Multi-node keycloak + template: + src: "standalone-ha.xml" + dest: "sunbird_auth_{{ sunbird_auth_version }}.config/standalone-ha.xml" + mode: 0750 + when: groups['keycloak'] | length > 1 + +- name: Keycloak configuration XML file - Single node keycloak template: - src: "roles/keycloak-deploy/templates/standalone-ha.xml" + src: "standalone-ha-single.xml" dest: "sunbird_auth_{{ sunbird_auth_version }}.config/standalone-ha.xml" mode: 0750 + when: groups['keycloak'] | length < 2 - name: Postgresql Configuration- module.xml template: @@ -129,7 +137,7 @@ when: "{{ msg_91_required|default(false) }}" ignore_errors: true -- name: Keycloak configuration XML file +- name: Keycloak configuration XML file copy: src: "sunbird_auth_{{ sunbird_auth_version }}.config/standalone-ha.xml" dest: "{{ keycloak_home }}/standalone/configuration/standalone-ha.xml" @@ -186,3 +194,12 @@ command: /etc/init.d/keycloak start poll: 5 async: 5 + +- name: get the list of all services in the system + become: yes + service_facts: + +- name: After succesful migration and staring of keycloak, start back monit if installed + service: name=monit state=started + become: yes + when: ansible_facts.services.monit is defined diff --git a/ansible/roles/keycloak-deploy/templates/keycloak-bootstrap.conf.j2 b/ansible/roles/keycloak-deploy/templates/keycloak-bootstrap.conf.j2 index 0e400efb8..71551589f 100644 --- a/ansible/roles/keycloak-deploy/templates/keycloak-bootstrap.conf.j2 +++ b/ansible/roles/keycloak-deploy/templates/keycloak-bootstrap.conf.j2 @@ -1,5 +1,5 @@ { - "keycloak_auth_server_url": "{{ keycloak_auth_server_url }}/", + "keycloak_auth_server_url": "http://localhost:8080/auth/", "keycloak_management_user": "{{ keycloak_management_user }}", "keycloak_management_password": "{{ keycloak_management_password }}", "keycloak_realm": "{{ keycloak_realm }}", diff --git a/ansible/roles/keycloak-deploy/templates/module.xml.j2 b/ansible/roles/keycloak-deploy/templates/module.xml.j2 index bcfe36a40..fc3c43c66 100644 --- a/ansible/roles/keycloak-deploy/templates/module.xml.j2 +++ b/ansible/roles/keycloak-deploy/templates/module.xml.j2 @@ -1,5 +1,5 @@ - + diff --git a/ansible/roles/keycloak-deploy/templates/standalone-ha-single.xml b/ansible/roles/keycloak-deploy/templates/standalone-ha-single.xml new file mode 100644 index 000000000..fb822740c --- /dev/null +++ b/ansible/roles/keycloak-deploy/templates/standalone-ha-single.xml @@ -0,0 +1,602 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE + h2 + + sa + sa + + + + jdbc:postgresql://{{keycloak_postgres_host}}:5432/{{ keycloak_postgres_database }}?sslmode=require + postgresql + + {{ keycloak_postgres_user }} + {{ keycloak_postgres_password }} + + + + + +org.postgresql.xa.PGXADataSource +org.postgresql.ds.PGPoolingDataSource + + + org.h2.jdbcx.JdbcDataSource + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + auth + + classpath:${jboss.home.dir}/providers/* + + master + 900 + + 2592000 + true + true + ${jboss.home.dir}/themes + + + + + + + + + + + + + jpa + + + basic + + + + + + + + + + + + + + + + + + + default + + + + + + + + ${keycloak.jta.lookup.provider:jboss} + + + + + + + + + + + ${keycloak.x509cert.lookup.provider:default} + + + + request + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ansible/roles/keycloak-deploy/templates/standalone-ha.xml b/ansible/roles/keycloak-deploy/templates/standalone-ha.xml index d705fd885..14ee65162 100644 --- a/ansible/roles/keycloak-deploy/templates/standalone-ha.xml +++ b/ansible/roles/keycloak-deploy/templates/standalone-ha.xml @@ -1,6 +1,6 @@ - + @@ -9,10 +9,8 @@ - - @@ -22,7 +20,12 @@ + + + + + @@ -39,6 +42,11 @@ + + + + + @@ -53,7 +61,7 @@ - + @@ -62,7 +70,8 @@ - + + @@ -77,7 +86,7 @@ - + @@ -92,20 +101,12 @@ - - - - - - - - - - - + + + @@ -117,7 +118,6 @@ - @@ -128,9 +128,10 @@ - + + - + jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE h2 @@ -138,41 +139,25 @@ sa - - jdbc:h2:${jboss.server.data.dir}/keycloak;AUTO_SERVER=TRUE - h2 - - sa - sa - - - - - jdbc:postgresql://thisnotwork:5432/keycloak + jdbc:postgresql://{{keycloak_postgres_host}}:5432/{{ keycloak_postgres_database }}?sslmode=require postgresql - - 100 - {{ keycloak_postgres_user }} {{ keycloak_postgres_password }} - - true - 10000 - - - + + + select 1 + false + true + 120000 + + + false + + org.h2.jdbcx.JdbcDataSource @@ -205,7 +190,7 @@ - + @@ -215,9 +200,8 @@ - - + @@ -233,7 +217,12 @@ - + + + + @@ -242,47 +231,50 @@ + - + - - + + - + - + - - - - + + + + + + - + - + - + - - + + - + - + @@ -290,28 +282,28 @@ - + - + - + - + - + - + - + @@ -330,8 +322,7 @@ - - + @@ -340,7 +331,7 @@ - + @@ -349,18 +340,15 @@ - + - + + - {{groups['keycloak']|join('[7600],')}}[7600] - + {{groups['keycloak'] | difference(ansible_default_ipv4.address) | join('[7600],')}}[7600] 10 - 3000 - 2 - @@ -372,6 +360,7 @@ + @@ -382,24 +371,22 @@ - - + - - + + - + - - + @@ -410,7 +397,107 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -427,11 +514,6 @@ - - - - - @@ -440,25 +522,35 @@ + + + + + - - + + + + - + + + + - + + - - + @@ -468,10 +560,6 @@ - - - - auth @@ -522,7 +610,7 @@ default - + @@ -537,6 +625,20 @@ + + ${keycloak.x509cert.lookup.provider:default} + + + + request + + + + + + + + @@ -556,12 +658,12 @@ - - - + + + + - - + diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 4c750937b..5f05e9a04 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -1,15 +1,17 @@ - --- +# Guideline for PR Merging: https://project-sunbird.atlassian.net/wiki/spaces/DevOps/pages/1210777612/API+Manager+Kong+BEST+Practices+Documentation. + +# OnboardAPI's: https://project-sunbird.atlassian.net/wiki/spaces/DevOps/pages/1205207131/Best+Practces+For+Onboarding+API+s+In+Sunbird + +# Onboard Consumers: https://project-sunbird.atlassian.net/wiki/spaces/DevOps/pages/1202847861/Best+Practces+For+Onboarding+Consumers+In+Sunbird kong_admin_api_url: "http://{{kong_host}}:8001" # Default statsd_metrics -statsd_metrics: ["request_count", "status_count", "latency", "upstream_latency", "request_size", "response_size"] -statsd_host: monitor_statsd_exporter +statsd_host: statsd-service statsd_port: 9125 statsd_pulgin: name: statsd - config.metrics: "{{ statsd_metrics }}" config.host: "{{ statsd_host }}" config.port: "{{ statsd_port }}" @@ -30,10 +32,17 @@ small_request_size_limit: 1 medium_request_size_limit: 10 large_request_size_limit: 100 +# External URL's +freshDesk_url: "http://dummy.freshDesk.url" + # Default service prefixes +learner_private_route_prefix: /private private_content_prefix: /private/content -content_service_prefix: /content +device_prefix: /v3/device +content_prefix: /content +tickets_prefix: /tickets course_service_prefix: /course +itemset_prefix: /itemset user_service_prefix: /user data_service_prefix: /data notes_service_prefix: /notes @@ -57,9 +66,23 @@ sso_service_prefix: /sso cert_service_prefix: /cert cert_registry_service_prefix: /certreg desktop_app_prefix: /desktop +license_api_prefix: /license +report_service_prefix: /data/v1/report-service +analytics_report_service_prefix: /report/jobs +group_service_prefix: /group +auth_service_prefix: /auth +object_category_prefix: /object/category +object_category_definition_prefix: /object/category/definition +dataset_service_prefix: /dataset +asset_prefix: /asset +collection_prefix: /collection +question_prefix: /question +questionset_prefix: /questionset +event_prefix: /event +notification_service_prefix: /notification # Service URLs -content_service_url: "http://knowledge-mw-service:5000" +knowledge_mw_service_url: "http://knowledge-mw-service:5000" learning_service_url: "http://learner-service:9000" vm_learning_service_url: "http://{{learningservice_ip}}:8080/learning-service" telemetry_service_url: "http://telemetry-service:9001" @@ -67,10 +90,16 @@ player_service_url: "http://player:3000" echo_service_url: "http://echo:9595" am_util_url: "http://adminutil:4000" config_service_url: "http://config-service:8080" -user_org_service_url: "http://user-org-service:9000" lms_service_url: "http://lms-service:9000" cert_service_url: "http://cert-service:9000" cert_registry_service_url: "http://cert-registry-service:9000" +content_service_url: "http://content-service:9000" +report_service_url: "http://report-service:3030" +group_service_url: "http://groups-service:9000" +analytics_api_service_url: "http://analytics-service:9000" +taxonomy_service_url: "http://taxonomy-service:9000" +assessment_service_url: "http://assessment-service:9000" +notification_service_url: "http://notification-service:9000" premium_consumer_rate_limits: - api: createContent @@ -86,940 +115,1043 @@ premium_consumer_rate_limits: config.limit_by: credential config.policy: local -# Example: -# kong_apis: -# - name: "readContent" -# request_path: "/v3/public/content/read" -# upstream_url: "{{ learning_service_url }}/v3/public/content/read" -# strip_request_path: true -# plugins: -# - {name: 'jwt'} -# - {name: 'cors'} -# - {name: 'statsd', config.metrics: "{{ statsd_metrics }}" } -# - {name: 'acl', config.whitelist: 'contentUser'} -# - {name: 'rate-limiting', config.hour: "{{ medium_rate_limit_per_hour }}"} -# - {name: 'request-size-limiting', config.allowed_payload_size: "{{ medium_request_size_limit }}" } kong_apis: - - name: PrivateContentAPIs - request_path: "{{ private_content_prefix }}/v3" - upstream_url: "{{ vm_learning_service_url }}/content/v3" - strip_request_path: true + + - name: acceptContentFlag + uris: "{{ content_prefix }}/v1/flag/accept" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/flag/accept" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: privateContentMgmt + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createContent - request_path: "{{ content_service_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/v1/content/create" - strip_request_path: true + + - name: acceptTermsAndCondition + uris: "{{ user_service_prefix }}/v1/tnc/accept" + upstream_url: "{{ learning_service_url }}/v1/user/tnc/accept" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchContent - request_path: "{{ content_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/content/search" - strip_request_path: true - plugins: - - name: cors - - "{{ statsd_pulgin }}" - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip - - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateContent - request_path: "{{ content_service_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/v1/content/update" - strip_request_path: true + + - name: addBadgeToUser + uris: "{{ user_service_prefix }}/v1/badges/add" + upstream_url: "{{ learning_service_url }}/v1/user/badges/add" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'badgeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readContent - request_path: "{{ content_service_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/v1/content/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: addMember + uris: "{{ org_service_prefix }}/v1/member/add" + upstream_url: "{{ learning_service_url }}/v1/org/member/add" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: submitContentForReview - request_path: "{{ content_service_prefix }}/v1/review" - upstream_url: "{{ content_service_url }}/v1/content/review" - strip_request_path: true + + - name: addSystemSettings + uris: "{{ data_service_prefix }}/v1/system/settings/set" + upstream_url: "{{ learning_service_url }}/v1/system/settings/set" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'appCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: publishContent - request_path: "{{ content_service_prefix }}/v1/publish" - upstream_url: "{{ content_service_url }}/v1/content/publish" - strip_request_path: true + + - name: addUserSkill + uris: "{{ user_service_prefix }}/v1/skill/add" + upstream_url: "{{ learning_service_url }}/v1/user/skill/add" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'userCreate ' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: uploadContent - request_path: "{{ content_service_prefix }}/v1/upload" - upstream_url: "{{ content_service_url }}/v1/content/upload" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: addUserSkillEndorsement + uris: "{{ user_service_prefix }}/v1/skill/endorse/add" + upstream_url: "{{ learning_service_url }}/v1/user/skill/endorse/add" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'userCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ large_request_size_limit }}" - - name: retireContent - request_path: "{{ content_service_prefix }}/v1/retire" - upstream_url: "{{ content_service_url }}/v1/content/retire" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: addUserToBatch + uris: "{{ course_service_prefix }}/v1/batch/user/add" + upstream_url: "{{ lms_service_url }}/v1/course/batch/users/add" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'courseCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createCourse - request_path: "{{ course_service_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/v1/course/create" - strip_request_path: true + + - name: assemblePage + uris: "{{ data_service_prefix }}/v1/page/assemble" + upstream_url: "{{ lms_service_url }}/v1/page/assemble" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: courseUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchCourse - request_path: "{{ course_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/course/search" - strip_request_path: true + + - name: assignRole + uris: "{{ user_service_prefix }}/v1/role/assign" + upstream_url: "{{ learning_service_url }}/v1/user/assign/role" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateCourse - request_path: "{{ course_service_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/v1/course/update" - strip_request_path: true + + - name: batchBulkEnrolment + uris: "{{ course_service_prefix }}/v1/batch/bulk/enrollment" + upstream_url: "{{ lms_service_url }}/v1/batch/bulk/enrollment" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUpdate + config.whitelist: + - 'courseSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readCourse - request_path: "{{ course_service_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/v1/course/read" - strip_request_path: true + + - name: batchBulkUnEnrolment + uris: "{{ course_service_prefix }}/v1/batch/bulk/unenrollment" + upstream_url: "{{ lms_service_url }}/v1/batch/bulk/unenrollment" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'courseSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: submitCourseForReview - request_path: "{{ course_service_prefix }}/v1/review" - upstream_url: "{{ content_service_url }}/v1/course/review" - strip_request_path: true + + - name: blockUser + uris: "{{ user_service_prefix }}/v1/block" + upstream_url: "{{ learning_service_url }}/v1/user/block" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUpdate + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: publishCourse - request_path: "{{ course_service_prefix }}/v1/publish" - upstream_url: "{{ content_service_url }}/v1/course/publish" - strip_request_path: true + + - name: checkUploadJobStatus + uris: "{{ data_service_prefix }}/v1/upload/status" + upstream_url: "{{ learning_service_url }}/v1/upload/status" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseAdmin + config.whitelist: + - 'orgSuperAdmin' + - 'userSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: retireCourse - request_path: "{{ course_service_prefix }}/v1/retire" - upstream_url: "{{ content_service_url }}/v1/course/retire" - strip_request_path: true + + - name: compositeSearch + uris: "{{ composite_service_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseAdmin + config.whitelist: + - 'contentAdmin' + - 'contentTempAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getCourseHierarchy - request_path: "{{ course_service_prefix }}/v1/hierarchy" - upstream_url: "{{ content_service_url }}/v1/course/hierarchy" - strip_request_path: true + + - name: contentBadgeLink + uris: "{{ badge_service_prefix }}/v1/content/link" + upstream_url: "{{ lms_service_url }}/v1/content/link" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'badgeUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: courseEnrolment - request_path: "{{ course_service_prefix }}/v1/enrol" - upstream_url: "{{ lms_service_url }}/v1/course/enroll" - strip_request_path: true + + - name: contentBadgeUnLink + uris: "{{ badge_service_prefix }}/v1/content/unlink" + upstream_url: "{{ lms_service_url }}/v1/content/unlink" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'badgeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: courseUnEnrolment - request_path: "{{ course_service_prefix }}/v1/unenrol" - upstream_url: "{{ lms_service_url }}/v1/course/unenroll" - strip_request_path: true + + - name: copyContent + uris: "{{ content_prefix }}/v1/copy" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/copy" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listCourseEnrollments - request_path: "{{ course_service_prefix }}/v1/user/enrollment/list" - upstream_url: "{{ lms_service_url }}/v1/user/courses/list" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: copyFramework + uris: "{{ framework_service_prefix }}/v1/copy" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/copy" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'frameworkCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readCourseEnrollment - request_path: "{{ course_service_prefix }}/v1/user/enrollment/read" - upstream_url: "{{ lms_service_url }}/v1/user/courses/read" - strip_request_path: true + + - name: courseBatchAddCertificateTemplate + uris: "{{ course_service_prefix }}/batch/cert/v1/template/add" + upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/template/add" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'certificateCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readContentState - request_path: "{{ course_service_prefix }}/v1/content/state/read" - upstream_url: "{{ lms_service_url }}/v1/content/state/read" - strip_request_path: true + + - name: courseBatchRemoveCertificateTemplate + uris: "{{ course_service_prefix }}/batch/cert/v1/template/remove" + upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/template/remove" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateContentState - request_path: "{{ course_service_prefix }}/v1/content/state/update" - upstream_url: "{{ lms_service_url }}/v1/content/state/update" - strip_request_path: true + + - name: courseEnrolment + uris: "{{ course_service_prefix }}/v1/enrol" + upstream_url: "{{ lms_service_url }}/v1/course/enroll" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createUser - request_path: "{{ user_service_prefix }}/v1/create" - upstream_url: "{{ learning_service_url }}/v1/user/create" - strip_request_path: true + + - name: courseIssueCertificate + uris: "{{ course_service_prefix }}/batch/cert/v1/issue" + upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/issue" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'certificateCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createUserVersion2 - request_path: "{{ user_service_prefix }}/v2/create" - upstream_url: "{{ learning_service_url }}/v2/user/create" - strip_request_path: true + + - name: courseUnEnrolment + uris: "{{ course_service_prefix }}/v1/unenrol" + upstream_url: "{{ lms_service_url }}/v1/course/unenroll" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateUser - request_path: "{{ user_service_prefix }}/v1/update" - upstream_url: "{{ learning_service_url }}/v1/user/update" - strip_request_path: true + + - name: createAssertion + uris: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/create" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'badgeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserProfile - request_path: "{{ user_service_prefix }}/v1/read" - upstream_url: "{{ learning_service_url }}/v1/user/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createBadgeClass + uris: "{{ badge_service_prefix }}/v1/issuer/badge/create" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'badgeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserProfileV2 - request_path: "{{ user_service_prefix }}/v2/read" - upstream_url: "{{ learning_service_url }}/v2/user/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createBatch + uris: "{{ course_service_prefix }}/v1/batch/create" + upstream_url: "{{ lms_service_url }}/v1/course/batch/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'courseCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchUser - request_path: "{{ user_service_prefix }}/v1/search" - upstream_url: "{{ learning_service_url }}/v1/user/search" - strip_request_path: true + + - name: createCertTemplate + uris: "{{ cert_service_prefix }}/v1/template/create" + upstream_url: "{{ cert_service_url }}/cert/v1/template/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'certificateCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: retireUser - request_path: "{{ user_service_prefix }}/v1/retire" - upstream_url: "{{ learning_service_url }}/v1/user/retire" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createChannel + uris: "{{ channel_service_prefix }}/v1/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/channel/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'channelCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createPage - request_path: "{{ data_service_prefix }}/v1/page/create" - upstream_url: "{{ lms_service_url }}/v1/page/create" - strip_request_path: true + + - name: createContent + uris: "{{ content_prefix }}/v1/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updatePage - request_path: "{{ data_service_prefix }}/v1/page/update" - upstream_url: "{{ lms_service_url }}/v1/page/update" - strip_request_path: true + + - name: createCourse + uris: "{{ course_service_prefix }}/v1/create" + upstream_url: "{{ lms_service_url }}/v1/course/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'courseCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getPageSettings - request_path: "{{ data_service_prefix }}/v1/page/read" - upstream_url: "{{ lms_service_url }}/v1/page/read" - strip_request_path: true + + - name: createData + uris: "{{ data_service_prefix }}/v1/object/create" + upstream_url: "{{ learning_service_url }}/v1/object/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'objectCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: dialAssemble - request_path: "{{ data_service_prefix }}/v1/dial/assemble" - upstream_url: "{{ lms_service_url }}/v1/page/assemble" - strip_request_path: true + config.allowed_payload_size: "{{ large_request_size_limit }}" + + - name: createForm + uris: "{{ data_service_prefix }}/v1/form/create" + upstream_url: "{{ player_service_url }}/plugin/v1/form/create" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'formCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: assemblePage - request_path: "{{ data_service_prefix }}/v1/page/assemble" - upstream_url: "{{ lms_service_url }}/v1/page/assemble" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createFramework + uris: "{{ framework_service_prefix }}/v1/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/create" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.hour: "{{ x_medium_rate_limit_per_hour }}" + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createNote - request_path: "{{ notes_service_prefix }}/v1/create" - upstream_url: "{{ learning_service_url }}/v1/note/create" - strip_request_path: true + + - name: createFrameworkCategory + uris: "{{ framework_service_prefix }}/v1/category/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/category/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: noteUpdate + config.whitelist: + - 'frameworkCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateNote - request_path: "{{ notes_service_prefix }}/v1/update" - upstream_url: "{{ learning_service_url }}/v1/note/update" - strip_request_path: true + + - name: createFrameworkTerm + uris: "{{ framework_service_prefix }}/v1/term/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/term/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: noteUpdate + config.whitelist: + - 'frameworkCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readNote - request_path: "{{ notes_service_prefix }}/v1/read" - upstream_url: "{{ learning_service_url }}/v1/note/read" - strip_request_path: true + + - name: createissuer + uris: "{{ badge_service_prefix }}/v1/issuer/create" + upstream_url: "{{ learning_service_url }}/v1/issuer/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: noteUser + config.whitelist: + - 'badgeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchNotes - request_path: "{{ notes_service_prefix }}/v1/search" - upstream_url: "{{ learning_service_url }}/v1/note/search" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createLocation + uris: "{{ org_service_prefix }}/v1/location/create" + upstream_url: "{{ learning_service_url }}/v1/notification/location/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: noteUser + config.whitelist: + - 'locationCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createOrg - request_path: "{{ org_service_prefix }}/v1/create" - upstream_url: "{{ learning_service_url }}/v1/org/create" - strip_request_path: true + + - name: createLock + uris: "{{ lock_service_prefix }}/v1/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/lock/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateOrg - request_path: "{{ org_service_prefix }}/v1/update" - upstream_url: "{{ learning_service_url }}/v1/org/update" - strip_request_path: true + + - name: createNote + uris: "{{ notes_service_prefix }}/v1/create" + upstream_url: "{{ learning_service_url }}/v1/note/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'noteCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readOrg - request_path: "{{ org_service_prefix }}/v1/read" - upstream_url: "{{ learning_service_url }}/v1/org/read" - strip_request_path: true + + - name: createOrg + uris: "{{ org_service_prefix }}/v1/create" + upstream_url: "{{ learning_service_url }}/v1/org/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'orgCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchOrg - request_path: "{{ org_service_prefix }}/v1/search" - upstream_url: "{{ learning_service_url }}/v1/org/search" - strip_request_path: true + + - name: createOrgType + uris: "{{ org_service_prefix }}/v1/type/create" + upstream_url: "{{ learning_service_url }}/v1/org/type/create" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: addMember - request_path: "{{ org_service_prefix }}/v1/member/add" - upstream_url: "{{ learning_service_url }}/v1/org/member/add" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createPage + uris: "{{ data_service_prefix }}/v1/page/create" + upstream_url: "{{ lms_service_url }}/v1/page/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'pageCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: removeMember - request_path: "{{ org_service_prefix }}/v1/member/remove" - upstream_url: "{{ learning_service_url }}/v1/org/member/remove" - strip_request_path: true + + - name: createPageSection + uris: "{{ data_service_prefix }}/v1/page/section/create" + upstream_url: "{{ lms_service_url }}/v1/page/section/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'pageCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: suspendMember - request_path: "{{ org_service_prefix }}/v1/member/suspend" - upstream_url: "{{ learning_service_url }}/v1/org/member/suspend" - strip_request_path: true + + - name: createPublisher + uris: "{{ dialcode_service_prefix }}/v1/publisher/create" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/publisher/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'dialcodeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateMembership - request_path: "{{ org_service_prefix }}/v1/member/update" - upstream_url: "{{ learning_service_url }}/v1/org/member/update" - strip_request_path: true + + - name: createSession + uris: "{{ sso_service_prefix }}/v1/create/session" + upstream_url: "{{ player_service_url }}/v1/sso/create/session" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'ssoAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: echo - request_path: "{{ echo_service_prefix }}" - upstream_url: "{{ echo_service_url }}" - strip_request_path: true + + - name: createUser + uris: "{{ user_service_prefix }}/v1/create" + upstream_url: "{{ learning_service_url }}/v1/user/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: echoUser + config.whitelist: + - 'userCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createPageSection - request_path: "{{ data_service_prefix }}/v1/page/section/create" - upstream_url: "{{ lms_service_url }}/v1/page/section/create" - strip_request_path: true + + - name: createUserLearnerV3 + uris: "{{ user_service_prefix }}/v1/signup" + upstream_url: "{{ learning_service_url }}/v1/user/signup" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updatePageSection - request_path: "{{ data_service_prefix }}/v1/page/section/update" - upstream_url: "{{ lms_service_url }}/v1/page/section/update" - strip_request_path: true + + - name: createUserVersion2 + uris: "{{ user_service_prefix }}/v2/create" + upstream_url: "{{ learning_service_url }}/v2/user/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listPageSections - request_path: "{{ data_service_prefix }}/v1/page/section/list" - upstream_url: "{{ lms_service_url }}/v1/page/section/list" - strip_request_path: true + + - name: createUserVersion3 + uris: "{{ user_service_prefix }}/v3/create" + upstream_url: "{{ learning_service_url }}/v3/user/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readPageSection - request_path: "{{ data_service_prefix }}/v1/page/section/read" - upstream_url: "{{ lms_service_url }}/v1/page/section/read" - strip_request_path: true + + - name: createUserVersion4 + uris: "{{ user_service_prefix }}/v4/create" + upstream_url: "{{ learning_service_url }}/v4/user/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readRoleMapping - request_path: "{{ data_service_prefix }}/v1/role/read" - upstream_url: "{{ learning_service_url }}/v1/role/read" - strip_request_path: true + + - name: deleteBadgeClass + uris: "{{ badge_service_prefix }}/v1/issuer/badge/delete" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'badgeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateAssessment - request_path: "{{ course_service_prefix }}/v1/assessment/update" - upstream_url: "{{ learning_service_url }}/v1/assessment/update" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: deleteData + uris: "{{ data_service_prefix }}/v1/object/delete" + upstream_url: "{{ learning_service_url }}/v1/object/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'objectAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readAssessmentResult - request_path: "{{ course_service_prefix }}/v1/assessment/result/read" - upstream_url: "{{ learning_service_url }}/v1/assessment/result/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: deleteIssuer + uris: "{{ badge_service_prefix }}/v1/issuer/delete" + upstream_url: "{{ learning_service_url }}/v1/issuer/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'badgeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateOrgStatus - request_path: "{{ org_service_prefix }}/v1/status/update" - upstream_url: "{{ learning_service_url }}/v1/org/status/update" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: deleteLocation + uris: "{{ org_service_prefix }}/v1/location/delete" + upstream_url: "{{ learning_service_url }}/v1/notification/location/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'locationAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readUserProfile - request_path: "{{ user_service_prefix }}/v1/profile/read" - upstream_url: "{{ learning_service_url }}/v1/user/getuser" - strip_request_path: true + + - name: deleteNote + uris: "{{ notes_service_prefix }}/v1/delete" + upstream_url: "{{ learning_service_url }}/v1/note/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'noteAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerMobileApp - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_app/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app/credential/register" - strip_request_path: true + + - name: deviceProfile + uris: "{{ device_prefix }}/profile" + upstream_url: "{{ analytics_api_service_url }}/v1/device/profile" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: mobileAdmin + config.whitelist: + - 'deviceCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerMobileDevice - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_device/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device/credential/register" - strip_request_path: true + + - name: deviceRegister + uris: "{{ device_prefix }}/register" + upstream_url: "{{ analytics_api_service_url }}/v1/device/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: mobileApp + config.whitelist: + - 'deviceCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -1027,33 +1159,31 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerMobileAppOpenRAP - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_app_openrap/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app_openrap/credential/register" - strip_request_path: true + - name: dialAssemble + uris: "{{ data_service_prefix }}/v1/dial/assemble" + upstream_url: "{{ lms_service_url }}/v1/dial/assemble" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: mobileAdminOpenRAP - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerMobileDeviceOpenRAP - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_device_openrap/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device_openrap/credential/register" - strip_request_path: true + - name: downloadCertificate + uris: "{{ user_service_prefix }}/v1/certs/download" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/download" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: mobileAppOpenRAP + config.whitelist: + - 'certificateAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -1061,1040 +1191,1168 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: compositeSearch - request_path: "{{ composite_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/search" - strip_request_path: true + - name: downloadRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/download" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/download" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'certificateAccess' - name: rate-limiting config.policy: local - config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listTerms - request_path: "{{ meta_service_prefix }}/v1/terms/list" - upstream_url: "{{ content_service_url }}/v1/terms/list" - strip_request_path: true + + - name: downloadTOC + uris: "{{ textbook_service_prefix }}/v1/toc/download" + upstream_url: "{{ lms_service_url }}/v1/textbook/toc/download" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: configUser + config.whitelist: + - 'contentAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listResourceBundles - request_path: "{{ meta_service_prefix }}/v1/resourcebundles/list" - upstream_url: "{{ content_service_url }}/v1/resourcebundles/list" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: echo + uris: "{{ echo_service_prefix }}" + upstream_url: "{{ echo_service_url }}" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: configUser + config.whitelist: + - 'ssoCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listOrdinals - request_path: "{{ meta_service_prefix }}/v1/ordinals/list" - upstream_url: "{{ content_service_url }}/v1/ordinals/list" - strip_request_path: true + + - name: experimentcreate + uris: "{{ data_service_prefix }}/v1/experiement/create" + upstream_url: "{{ analytics_api_service_url }}/experiment/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: configUser + config.whitelist: + - 'experimentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: telemetry - request_path: "{{ data_service_prefix }}/v1/telemetry" - upstream_url: "{{ telemetry_service_url }}/v1/telemetry" - strip_request_path: true + + - name: experimentread + uris: "{{ data_service_prefix }}/v1/experiement/get" + upstream_url: "{{ analytics_api_service_url }}/experiment/get" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'expirementAccess' - name: rate-limiting config.policy: local - config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: flagContent - request_path: "{{ content_service_prefix }}/v1/flag" - upstream_url: "{{ content_service_url }}/v1/content/flag" - strip_request_path: true + + - name: fetchURLMetaInformation + uris: "{{ content_prefix }}/v1/fetchmeta" + upstream_url: "{{ knowledge_mw_service_url }}/v1/url/fetchmeta" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: acceptContentFlag - request_path: "{{ content_service_prefix }}/v1/flag/accept" - upstream_url: "{{ content_service_url }}/v1/content/flag/accept" - strip_request_path: true + + - name: flagContent + uris: "{{ content_prefix }}/v1/flag" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/flag" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'contentAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: rejectContentFlag - request_path: "{{ content_service_prefix }}/v1/flag/reject" - upstream_url: "{{ content_service_url }}/v1/content/flag/reject" - strip_request_path: true + + - name: freshDesk + uris: "{{ tickets_prefix }}/v1/create" + upstream_url: "{{ freshDesk_url }}/api/v2/tickets" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getOrgCreationDashboard - request_path: "{{ dashboard_service_prefix }}/v1/creation/org" - upstream_url: "{{ learning_service_url }}/v1/dashboard/creation/org" - strip_request_path: true + + - name: generateDialcodes + uris: "{{ dialcode_service_prefix }}/v1/generate" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/generate" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'dialcodeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getOrgConsumptionDashboard - request_path: "{{ dashboard_service_prefix }}/v1/consumption/org" - upstream_url: "{{ learning_service_url }}/v1/dashboard/consumption/org" - strip_request_path: true + + - name: generateOtp + uris: "{{ otp_service_prefix }}/v1/generate" + upstream_url: "{{ learning_service_url }}/v1/otp/generate" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getCourseProgressDashboard - request_path: "{{ dashboard_service_prefix }}/v1/progress/course" - upstream_url: "{{ lms_service_url }}/v1/dashboard/progress/course" - strip_request_path: true + + - name: getAllData + uris: "{{ data_service_prefix }}/v1/object/read/list" + upstream_url: "{{ learning_service_url }}/v1/object/read/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'objectAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getCourseProgressDashboardV2 - request_path: "{{ dashboard_service_prefix }}/v2/progress/course" - upstream_url: "{{ lms_service_url }}/v2/dashboard/progress/course" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getAllIssuer + uris: "{{ badge_service_prefix }}/v1/issuer/list" + upstream_url: "{{ learning_service_url }}/v1/issuer/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getCourseConsumptionDashboard - request_path: "{{ dashboard_service_prefix }}/v1/consumption/course" - upstream_url: "{{ lms_service_url }}/v1/dashboard/consumption/course" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getAssertion + uris: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/read" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserCreationDashboard - request_path: "{{ dashboard_service_prefix }}/v1/creation/user" - upstream_url: "{{ learning_service_url }}/v1/dashboard/creation/user" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getAssertionList + uris: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/search" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserConsumptionDasbhoard - request_path: "{{ dashboard_service_prefix }}/v1/consumption/user" - upstream_url: "{{ learning_service_url }}/v1/dashboard/consumption/user" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getAudienceCount + uris: "{{ data_service_prefix }}/v1/notification/audience" + upstream_url: "{{ learning_service_url }}/v1/notification/audience" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'announcementAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: userBulkUpload - request_path: "{{ user_service_prefix }}/v1/upload" - upstream_url: "{{ learning_service_url }}/v1/user/upload" - strip_request_path: true + + - name: getBadgeClass + uris: "{{ badge_service_prefix }}/v1/issuer/badge/read" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: orgBulkUpload - request_path: "{{ org_service_prefix }}/v1/upload" - upstream_url: "{{ learning_service_url }}/v1/org/upload" - strip_request_path: true + + - name: getBatch + uris: "{{ course_service_prefix }}/v1/batch/read" + upstream_url: "{{ lms_service_url }}/v1/course/batch/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: checkUploadJobStatus - request_path: "{{ data_service_prefix }}/v1/upload/status" - upstream_url: "{{ learning_service_url }}/v1/upload/status" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: collectionSummaryAgg + uris: "/v1/collection/summary" + upstream_url: "{{ lms_service_url }}/v1/collection/summary" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appAdmin + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUploadJobStatusLink - request_path: "{{ data_service_prefix }}/v1/upload/statusDownloadLink" - upstream_url: "{{ learning_service_url }}/v1/upload/statusDownloadLink" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: getContentUploadUrl + uris: "{{ content_prefix }}/v1/upload/url/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/upload/url" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appAdmin + config.whitelist: + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createBatch - request_path: "{{ course_service_prefix }}/v1/batch/create" - upstream_url: "{{ lms_service_url }}/v1/course/batch/create" - strip_request_path: true + + - name: getCourseHierarchy + uris: "{{ course_service_prefix }}/v1/hierarchy" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/hierarchy" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: courseUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateBatch - request_path: "{{ course_service_prefix }}/v1/batch/update" - upstream_url: "{{ lms_service_url }}/v1/course/batch/update" - strip_request_path: true + + - name: getData + uris: "{{ data_service_prefix }}/v1/object/read" + upstream_url: "{{ learning_service_url }}/v1/object/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUpdate + config.whitelist: + - 'objectAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: addUserToBatch - request_path: "{{ course_service_prefix }}/v1/batch/user/add" - upstream_url: "{{ lms_service_url }}/v1/course/batch/users/add" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getIssuerDetails + uris: "{{ badge_service_prefix }}/v1/issuer/read" + upstream_url: "{{ learning_service_url }}/v1/issuer/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUpdate + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: removeUserFromBatch - request_path: "{{ course_service_prefix }}/v1/batch/user/remove" - upstream_url: "{{ lms_service_url }}/v1/course/batch/users/remove" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getLocation + uris: "{{ org_service_prefix }}/v1/location/read" + upstream_url: "{{ learning_service_url }}/v1/notification/location/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUpdate + config.whitelist: + - 'locationAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getBatch - request_path: "{{ course_service_prefix }}/v1/batch/read" - upstream_url: "{{ lms_service_url }}/v1/course/batch/read" - strip_request_path: true + + - name: getMediaTypes + uris: "{{ user_service_prefix }}/v1/mediatype/list" + upstream_url: "{{ learning_service_url }}/v1/user/mediatype/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getParticipants - request_path: "{{ course_service_prefix }}/v1/batch/participants/list" - upstream_url: "{{ lms_service_url }}/v1/batch/participants/list" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getPageSettings + uris: "{{ data_service_prefix }}/v1/page/read" + upstream_url: "{{ lms_service_url }}/v1/page/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'pageAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: assignRole - request_path: "{{ user_service_prefix }}/v1/role/assign" - upstream_url: "{{ learning_service_url }}/v1/user/assign/role" - strip_request_path: true + + - name: getParticipants + uris: "{{ course_service_prefix }}/v1/batch/participants/list" + upstream_url: "{{ lms_service_url }}/v1/batch/participants/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'courseCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: indexSync - request_path: "{{ data_service_prefix }}/v1/index/sync" - upstream_url: "{{ learning_service_url }}/v1/data/sync" - strip_request_path: true + + - name: getSkills + uris: "{{ data_service_prefix }}/v1/skills" + upstream_url: "{{ learning_service_url }}/v1/skills" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: blockUser - request_path: "{{ user_service_prefix }}/v1/block" - upstream_url: "{{ learning_service_url }}/v1/user/block" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getSystemSettings + uris: "{{ data_service_prefix }}/v1/system/settings/get" + upstream_url: "{{ learning_service_url }}/v1/system/settings/get" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: unblockUser - request_path: "{{ user_service_prefix }}/v1/unblock" - upstream_url: "{{ learning_service_url }}/v1/user/unblock" - strip_request_path: true + + - name: getTenantInfo + uris: "{{ org_service_prefix }}/v1/tenant/info" + upstream_url: "{{ player_service_url }}/v1/tenant/info" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'orgAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: uploadMedia - request_path: "{{ content_service_prefix }}/v1/media/upload" - upstream_url: "{{ learning_service_url }}/v1/file/upload" - strip_request_path: true + + - name: getUploadJobStatusLink + uris: "{{ data_service_prefix }}/v1/upload/statusDownloadLink" + upstream_url: "{{ learning_service_url }}/v1/upload/statusDownloadLink" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'orgSuperAdmin' + - 'userSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: listCourseBatches - request_path: "{{ course_service_prefix }}/v1/batch/list" - upstream_url: "{{ lms_service_url }}/v1/course/batch/search" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: getUserByKey + uris: "{{ user_service_prefix }}/v1/get" + upstream_url: "{{ learning_service_url }}/v1/user/get" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: batchBulkEnrolment - request_path: "{{ course_service_prefix }}/v1/batch/bulk/enrollment" - upstream_url: "{{ lms_service_url }}/v1/batch/bulk/enrollment" - strip_request_path: true + + - name: getUserConsumptionDasbhoard + uris: "{{ dashboard_service_prefix }}/v1/consumption/user" + upstream_url: "{{ learning_service_url }}/v1/dashboard/consumption/user" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: batchBulkUnEnrolment - request_path: "{{ course_service_prefix }}/v1/batch/bulk/unenrollment" - upstream_url: "{{ lms_service_url }}/v1/batch/bulk/unenrollment" - strip_request_path: true + + - name: getUserProfile + uris: "{{ user_service_prefix }}/v1/read" + upstream_url: "{{ learning_service_url }}/v1/user/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getTenantInfo - request_path: "{{ org_service_prefix }}/v1/tenant/info" - upstream_url: "{{ player_service_url }}/v1/tenant/info" - strip_request_path: true + + - name: getUserProfileV2 + uris: "{{ user_service_prefix }}/v2/read" + upstream_url: "{{ learning_service_url }}/v2/user/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getContentUploadUrl - request_path: "{{ content_service_prefix }}/v1/upload/url/read" - upstream_url: "{{ content_service_url }}/v1/content/upload/url" - strip_request_path: true + + - name: getUserSkill + uris: "{{ user_service_prefix }}/v1/skill/read" + upstream_url: "{{ learning_service_url }}/v1/user/skill/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: rejectContent - request_path: "{{ content_service_prefix }}/v1/reject" - upstream_url: "{{ content_service_url }}/v1/content/reject" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getUserType + uris: "{{ user_service_prefix }}/v1/type/list" + upstream_url: "{{ learning_service_url }}/v1/user/type/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: deleteNote - request_path: "{{ notes_service_prefix }}/v1/delete" - upstream_url: "{{ learning_service_url }}/v1/note/delete" - strip_request_path: true + + - name: indexSync + uris: "{{ data_service_prefix }}/v1/index/sync" + upstream_url: "{{ learning_service_url }}/v1/data/sync" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: noteAdmin + config.whitelist: + - 'userAdmin' + - 'orgAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: sendEmailNotification - request_path: "{{ user_service_prefix }}/v1/notification/email" - upstream_url: "{{ learning_service_url }}/v1/notification/email" - strip_request_path: true + + - name: ItemSetAPIs + uris: "{{ itemset_prefix }}/v1" + upstream_url: "{{ knowledge_mw_service_url }}/itemset/v1" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'itemSetAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: listAllBadges - request_path: "{{ org_service_prefix }}/v1/badges/list" - upstream_url: "{{ learning_service_url }}/v1/badges/list" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: KongConsumerApi + uris: "{{ api_manager_perfix }}/v1/consumer" + upstream_url: "{{ am_util_url }}/v1/consumer" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'kongConsumerAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: addBadgeToUser - request_path: "{{ user_service_prefix }}/v1/badges/add" - upstream_url: "{{ learning_service_url }}/v1/user/badges/add" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: LicenseAPI + uris: "{{ license_api_prefix }}/v3" + upstream_url: "{{ content_service_url }}/license/v3" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'appAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: listOrgTypes - request_path: "{{ org_service_prefix }}/v1/type/list" - upstream_url: "{{ learning_service_url }}/v1/org/type/list" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: linkDialcodeContent + uris: "{{ dialcode_service_prefix }}/v1/content/link" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/content/link" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createOrgType - request_path: "{{ org_service_prefix }}/v1/type/create" - upstream_url: "{{ learning_service_url }}/v1/org/type/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listAllBadges + uris: "{{ org_service_prefix }}/v1/badges/list" + upstream_url: "{{ learning_service_url }}/v1/badges/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateOrgType - request_path: "{{ org_service_prefix }}/v1/type/update" - upstream_url: "{{ learning_service_url }}/v1/org/type/update" - strip_request_path: true + + - name: listChannel + uris: "{{ channel_service_prefix }}/v1/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/channel/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'channelAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateUserloginTime - request_path: "{{ user_service_prefix }}/v1/update/logintime" - upstream_url: "{{ learning_service_url }}/v1/user/update/logintime" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchCourseBatches + uris: "{{ course_service_prefix }}/v1/batch/list" + upstream_url: "{{ lms_service_url }}/v1/course/batch/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getMediaTypes - request_path: "{{ user_service_prefix }}/v1/mediatype/list" - upstream_url: "{{ learning_service_url }}/v1/user/mediatype/list" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listCourseEnrollments + uris: "{{ course_service_prefix }}/v1/user/enrollment/list" + upstream_url: "{{ lms_service_url }}/v1/user/courses/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: DecryptData - request_path: "{{ data_service_prefix }}/v1/user/decrypt" - upstream_url: "{{ learning_service_url }}/v1/user/data/decrypt" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listUserCourseEnrollments + uris: "{{ course_service_prefix }}/v2/user/enrollment/list" + upstream_url: "{{ lms_service_url }}/v2/user/courses/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: encryptData - request_path: "{{ data_service_prefix }}/v1/user/encrypt" - upstream_url: "{{ learning_service_url }}/v1/user/data/encrypt" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + + - name: listDialcodes + uris: "{{ dialcode_service_prefix }}/v1/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'dialcodeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: searchAuditHistory - request_path: "{{ data_service_prefix }}/v1/audit/history" - upstream_url: "{{ learning_service_url }}/v1/audit/history" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listFramework + uris: "{{ framework_service_prefix }}/v1/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appAdmin + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ x_medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: addUserSkill - request_path: "{{ user_service_prefix }}/v1/skill/add" - upstream_url: "{{ learning_service_url }}/v1/user/skill/add" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listLock + uris: "{{ lock_service_prefix }}/v1/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/lock/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getUserSkill - request_path: "{{ user_service_prefix }}/v1/skill/read" - upstream_url: "{{ learning_service_url }}/v1/user/skill/read" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listOrdinals + uris: "{{ meta_service_prefix }}/v1/ordinals/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/ordinals/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getSkills - request_path: "{{ data_service_prefix }}/v1/skills" - upstream_url: "{{ learning_service_url }}/v1/skills" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listOrgTypes + uris: "{{ org_service_prefix }}/v1/type/list" + upstream_url: "{{ learning_service_url }}/v1/org/type/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'orgAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateUserSkill - request_path: "{{ user_service_prefix }}/v1/skill/update" - upstream_url: "{{ learning_service_url }}/v1/user/skill/update" - strip_request_path: true + + - name: listPageSections + uris: "{{ data_service_prefix }}/v1/page/section/list" + upstream_url: "{{ lms_service_url }}/v1/page/section/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'pageAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: addUserSkillEndorsement - request_path: "{{ user_service_prefix }}/v1/skill/endorse/add" - upstream_url: "{{ learning_service_url }}/v1/user/skill/endorse/add" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listResourceBundles + uris: "{{ meta_service_prefix }}/v1/resourcebundles/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/resourcebundles/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getAllData - request_path: "{{ data_service_prefix }}/v1/object/read/list" - upstream_url: "{{ learning_service_url }}/v1/object/read/list" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listSystemSettings + uris: "{{ data_service_prefix }}/v1/system/settings/list" + upstream_url: "{{ learning_service_url }}/v1/system/settings/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getData - request_path: "{{ data_service_prefix }}/v1/object/read" - upstream_url: "{{ learning_service_url }}/v1/object/read" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listTerms + uris: "{{ meta_service_prefix }}/v1/terms/list" + upstream_url: "{{ knowledge_mw_service_url }}/v1/terms/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createData - request_path: "{{ data_service_prefix }}/v1/object/create" - upstream_url: "{{ learning_service_url }}/v1/object/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: masterLocationCreate + uris: "{{ data_service_prefix }}/v1/location/create" + upstream_url: "{{ learning_service_url }}/v1/location/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'locationCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ large_request_size_limit }}" - - name: updateData - request_path: "{{ data_service_prefix }}/v1/object/update" - upstream_url: "{{ learning_service_url }}/v1/object/update" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: masterLocationDelete + uris: "{{ data_service_prefix }}/v1/location/delete" + upstream_url: "{{ learning_service_url }}/v1/location/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'locationAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: deleteData - request_path: "{{ data_service_prefix }}/v1/object/delete" - upstream_url: "{{ learning_service_url }}/v1/object/delete" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: masterLocationSearch + uris: "{{ data_service_prefix }}/v1/location/search" + upstream_url: "{{ learning_service_url }}/v1/location/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataAdmin + config.whitelist: + - 'locationAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: searchData - request_path: "{{ data_service_prefix }}/v1/object/search" - upstream_url: "{{ learning_service_url }}/v1/object/search" - strip_request_path: true + + - name: masterLocationUpdate + uris: "{{ data_service_prefix }}/v1/location/update" + upstream_url: "{{ learning_service_url }}/v1/location/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'locationUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getAnnouncementDefinitions - request_path: "{{announcement_service_prefix}}/v1/definitions" - upstream_url: "{{ player_service_url }}/announcement/v1/definitions" - strip_request_path: true + + - name: masterLocationUpload + uris: "{{ data_service_prefix }}/v1/bulk/location/upload" + upstream_url: "{{ learning_service_url }}/v1/bulk/location/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'locationSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createAnnouncement - request_path: "{{announcement_service_prefix}}/v1/create" - upstream_url: "{{ player_service_url }}/announcement/v1/create" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: mergeUserAccounts + uris: "{{ user_service_prefix }}/v1/account/merge" + upstream_url: "{{ learning_service_url }}/private/user/v1/account/merge" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: cancelAnnouncement - request_path: "{{announcement_service_prefix}}/v1/cancel" - upstream_url: "{{ player_service_url }}/announcement/v1/cancel" - strip_request_path: true + + - name: metricsSearchData + uris: "{{ data_service_prefix }}/v1/object/metrics" + upstream_url: "{{ learning_service_url }}/v1/object/metrics" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'objectAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getAnnouncementInbox - request_path: "{{announcement_service_prefix}}/v1/user/inbox" - upstream_url: "{{ player_service_url }}/announcement/v1/user/inbox" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: orgAssignKeys + uris: "{{ org_service_prefix }}/v1/assign/key" + upstream_url: "{{ learning_service_url }}/v1/org/assign/key" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'orgAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getAnnouncementOutbox - request_path: "{{announcement_service_prefix}}/v1/user/outbox" - upstream_url: "{{ player_service_url }}/announcement/v1/user/outbox" - strip_request_path: true + + - name: orgBulkUpload + uris: "{{ org_service_prefix }}/v1/upload" + upstream_url: "{{ learning_service_url }}/v1/org/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'orgSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getAnnouncement - request_path: "{{announcement_service_prefix}}/v1/get" - upstream_url: "{{ player_service_url }}/announcement/v1/get" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: pluginSearch + uris: "{{ plugin_service_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/plugins/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'pluginAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -2102,1401 +2360,1572 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerClient - request_path: "{{ data_service_prefix }}/v1/client/register" - upstream_url: "{{ learning_service_url }}/v1/client/register" - strip_request_path: true + - name: PrivateContentCreateAPIs + uris: "{{ private_content_prefix }}/v3/create" + upstream_url: "{{ content_service_url }}/content/v3/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataAdmin + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateClientKey - request_path: "{{ data_service_prefix }}/v1/client/key/update" - upstream_url: "{{ learning_service_url }}/v1/client/key/update" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: PrivateContentReadAPIs + uris: "{{ private_content_prefix }}/v3/read" + upstream_url: "{{ content_service_url }}/content/v3/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getClientKey - request_path: "{{ data_service_prefix }}/v1/client/key/read" - upstream_url: "{{ learning_service_url }}/v1/client/key/read" - strip_request_path: true + + - name: PrivateContentUpdateAPIs + uris: "{{ private_content_prefix }}/v3/update" + upstream_url: "{{ content_service_url }}/content/v3/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getTenantPreference - request_path: "{{ org_service_prefix }}/v1/tc/read" - upstream_url: "{{ learning_service_url }}/v1/org/tc/read" - strip_request_path: true + + - name: PrivateContentRetireAPI + uris: "{{ private_content_prefix }}/v3/retire" + upstream_url: "{{ vm_learning_service_url }}/content/v3/retire" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createTenantPreference - request_path: "{{ org_service_prefix }}/v1/tc/create" - upstream_url: "{{ learning_service_url }}/v1/org/tc/create" - strip_request_path: true + + - name: PrivateContentUploadAPI + uris: "{{ private_content_prefix }}/v3/upload" + upstream_url: "{{ content_service_url }}/content/v3/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateTenantPreference - request_path: "{{ org_service_prefix }}/v1/tc/update" - upstream_url: "{{ learning_service_url }}/v1/org/tc/update" - strip_request_path: true + + - name: PrivateContentReviewAPI + uris: "{{ private_content_prefix }}/v3/review" + upstream_url: "{{ vm_learning_service_url }}/content/v3/review" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateUserTcStatus - request_path: "{{ user_service_prefix }}/v1/tc/update" - upstream_url: "{{ learning_service_url }}/v1/user/tc/update" - strip_request_path: true + + - name: PrivateContentPublishAPI + uris: "{{ private_content_prefix }}/v3/publish" + upstream_url: "{{ vm_learning_service_url }}/content/v3/publish" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createLocation - request_path: "{{ org_service_prefix }}/v1/location/create" - upstream_url: "{{ learning_service_url }}/v1/notification/location/create" - strip_request_path: true + + - name: PrivateContentRejectAPI + uris: "{{ private_content_prefix }}/v3/reject" + upstream_url: "{{ vm_learning_service_url }}/content/v3/reject" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getLocation - request_path: "{{ org_service_prefix }}/v1/location/read" - upstream_url: "{{ learning_service_url }}/v1/notification/location/read" - strip_request_path: true + + - name: PrivateContentHierarchyAPI + uris: "{{ private_content_prefix }}/v3/hierarchy" + upstream_url: "{{ content_service_url }}/content/v3/hierarchy" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'contentSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: deleteLocation - request_path: "{{ org_service_prefix }}/v1/location/delete" - upstream_url: "{{ learning_service_url }}/v1/notification/location/delete" - strip_request_path: true + + - name: privateUserAssignRole + uris: "{{ user_service_prefix }}/private/v1/assign/role" + upstream_url: "{{ learning_service_url }}/private/user/v1/assign/role" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'userSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateLocation - request_path: "{{ org_service_prefix }}/v1/location/update" - upstream_url: "{{ learning_service_url }}/v1/notification/location/update" - strip_request_path: true + + - name: privateUserRead + uris: "{{ user_service_prefix }}/private/v1/read" + upstream_url: "{{ learning_service_url }}/private/user/v1/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'userSuperAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: profileVisibility - request_path: "{{ user_service_prefix }}/v1/profile/visibility" - upstream_url: "{{ learning_service_url }}/v1/user/profile/visibility" - strip_request_path: true + + - name: privateUserUpdate + uris: "{{ user_service_prefix }}/private/v1/update" + upstream_url: "{{ learning_service_url }}/private/user/v1/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userSuperAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: sendNotification - request_path: "{{ data_service_prefix }}/v1/notification/send" - upstream_url: "{{ learning_service_url }}/v1/notification/send" - strip_request_path: true + + - name: profileVisibility + uris: "{{ user_service_prefix }}/v1/profile/visibility" + upstream_url: "{{ learning_service_url }}/v1/user/profile/visibility" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: announcementResend - request_path: "{{ announcement_service_prefix }}/v1/resend" - upstream_url: "{{ player_service_url }}/announcement/v1/resend" - strip_request_path: true + + - name: publishContent + uris: "{{ content_prefix }}/v1/publish" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/publish" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: announcementReceived - request_path: "{{ announcement_service_prefix }}/v1/received" - upstream_url: "{{ player_service_url }}/announcement/v1/received" - strip_request_path: true + + - name: publishCourse + uris: "{{ course_service_prefix }}/v1/publish" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/publish" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: announcementRead - request_path: "{{ announcement_service_prefix }}/v1/read" - upstream_url: "{{ player_service_url }}/announcement/v1/read" - strip_request_path: true + + - name: publishDialcode + uris: "{{ dialcode_service_prefix }}/v1/publish" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/publish" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: metricsSearchData - request_path: "{{ data_service_prefix }}/v1/object/metrics" - upstream_url: "{{ learning_service_url }}/v1/object/metrics" - strip_request_path: true + + - name: publishFramework + uris: "{{ framework_service_prefix }}/v1/publish" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/publish" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'frameworkAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateSystemSettings - request_path: "{{ data_service_prefix }}/v1/system/settings/list" - upstream_url: "{{ learning_service_url }}/v1/system/settings/list" - strip_request_path: true + + - name: qrCodeBatchProcessRetry + uris: "{{ dialcode_service_prefix }}/v1/process/retry" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/process/retry" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getAudienceCount - request_path: "{{ data_service_prefix }}/v1/notification/audience" - upstream_url: "{{ learning_service_url }}/v1/notification/audience" - strip_request_path: true + + - name: qrCodeBatchProcessStatus + uris: "{{ dialcode_service_prefix }}/v1/process/status" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/process/status" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'dialcodeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: generateDialcodes - request_path: "{{ dialcode_service_prefix }}/v1/generate" - upstream_url: "{{ content_service_url }}/v1/dialcode/generate" - strip_request_path: true + + - name: qrCodeDownload + uris: "{{ course_service_prefix }}/v1/qrcode/download" + upstream_url: "{{ lms_service_url }}/v1/course/qrcode/download" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'dialcodeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readDialcode - request_path: "{{ dialcode_service_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/v1/dialcode/read" - strip_request_path: true + + - name: readCertTemplate + uris: "{{ cert_service_prefix }}/v1/template/read" + upstream_url: "{{ cert_service_url }}/cert/v1/template/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUser + config.whitelist: + - 'certificateAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateDialcode - request_path: "{{ dialcode_service_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/v1/dialcode/update" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readChannel + uris: "{{ channel_service_prefix }}/v1/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/channel/read" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: dialcodeUpdate - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.hour: "{{ x_large_rate_limit_per_hour }}" + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listDialcodes - request_path: "{{ dialcode_service_prefix }}/v1/list" - upstream_url: "{{ content_service_url }}/v1/dialcode/list" - strip_request_path: true + + - name: readContent + uris: "{{ content_prefix }}/v1/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/read" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: dialcodeUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: publishDialcode - request_path: "{{ dialcode_service_prefix }}/v1/publish" - upstream_url: "{{ content_service_url }}/v1/dialcode/publish" - strip_request_path: true + + - name: readContentState + uris: "{{ course_service_prefix }}/v1/content/state/read" + upstream_url: "{{ lms_service_url }}/v1/content/state/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeAdmin + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchDialcodes - request_path: "{{ dialcode_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/dialcode/search" - strip_request_path: true + + - name: readCourse + uris: "{{ course_service_prefix }}/v1/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: qrCodeBatchProcessStatus - request_path: "{{ dialcode_service_prefix }}/v1/process/status" - upstream_url: "{{ content_service_url }}/v1/dialcode/process/status" - strip_request_path: true + + - name: readCourseEnrollment + uris: "{{ course_service_prefix }}/v1/user/enrollment/read" + upstream_url: "{{ lms_service_url }}/v1/user/courses/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: qrCodeBatchProcessRetry - request_path: "{{ dialcode_service_prefix }}/v1/process/retry" - upstream_url: "{{ content_service_url }}/v1/dialcode/process/retry" - strip_request_path: true + + - name: readDialcode + uris: "{{ dialcode_service_prefix }}/v1/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'dialcodeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createPublisher - request_path: "{{ dialcode_service_prefix }}/v1/publisher/create" - upstream_url: "{{ content_service_url }}/v1/dialcode/publisher/create" - strip_request_path: true + + - name: readForm + uris: "{{ data_service_prefix }}/v1/form/read" + upstream_url: "{{ player_service_url }}/plugin/v1/form/read" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: dialcodeUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readFramework + uris: "{{ framework_service_prefix }}/v1/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/read" + strip_uri: true + plugins: + - name: cors + - "{{ statsd_pulgin }}" + - name: rate-limiting + config.policy: local + config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updatePublisher - request_path: "{{ dialcode_service_prefix }}/v1/publisher/update" - upstream_url: "{{ content_service_url }}/v1/dialcode/publisher/update" - strip_request_path: true + + - name: readFrameworkCategory + uris: "{{ framework_service_prefix }}/v1/category/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/category/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readPublisher - request_path: "{{ dialcode_service_prefix }}/v1/publisher/read" - upstream_url: "{{ content_service_url }}/v1/dialcode/publisher/read" - strip_request_path: true + + - name: readFrameworkTerm + uris: "{{ framework_service_prefix }}/v1/term/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/term/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ x_large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: linkDialcodeContent - request_path: "{{ dialcode_service_prefix }}/v1/content/link" - upstream_url: "{{ content_service_url }}/v1/dialcode/content/link" - strip_request_path: true + + - name: readNote + uris: "{{ notes_service_prefix }}/v1/read" + upstream_url: "{{ learning_service_url }}/v1/note/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeUpdate + config.whitelist: + - 'noteAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readChannel - request_path: "{{ channel_service_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/v1/channel/read" - strip_request_path: true + + - name: readOrg + uris: "{{ org_service_prefix }}/v1/read" + upstream_url: "{{ learning_service_url }}/v1/org/read" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgAccess' - name: rate-limiting config.policy: local - config.hour: "{{ x_large_rate_limit_per_hour }}" - config.limit_by: ip + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listChannel - request_path: "{{ channel_service_prefix }}/v1/list" - upstream_url: "{{ content_service_url }}/v1/channel/list" - strip_request_path: true + + - name: readPageSection + uris: "{{ data_service_prefix }}/v1/page/section/read" + upstream_url: "{{ lms_service_url }}/v1/page/section/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'pageAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchChannel - request_path: "{{ channel_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/channel/search" - strip_request_path: true + + - name: readPublisher + uris: "{{ dialcode_service_prefix }}/v1/publisher/read" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/publisher/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'dialcodeCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createChannel - request_path: "{{ channel_service_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/v1/channel/create" - strip_request_path: true + + - name: readRoleMapping + uris: "{{ data_service_prefix }}/v1/role/read" + upstream_url: "{{ learning_service_url }}/v1/role/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateChannel - request_path: "{{ channel_service_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/v1/channel/update" - strip_request_path: true + + - name: readUserProfile + uris: "{{ user_service_prefix }}/v1/profile/read" + upstream_url: "{{ learning_service_url }}/v1/user/getuser" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readFramework - request_path: "{{ framework_service_prefix }}/v1/read" - upstream_url: "{{ content_service_url }}/v1/framework/read" - strip_request_path: true + + - name: registerDesktopApp + uris: "{{ api_manager_perfix }}/v1/consumer/desktop_app/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/desktop_app/credential/register" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'desktopSuperAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" - config.limit_by: ip + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listFramework - request_path: "{{ framework_service_prefix }}/v1/list" - upstream_url: "{{ content_service_url }}/v1/framework/list" - strip_request_path: true + + - name: registerDesktopDevice + uris: "{{ api_manager_perfix }}/v1/consumer/desktop_device/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/desktop_device/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'desktopAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ x_medium_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createFramework - request_path: "{{ framework_service_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/v1/framework/create" - strip_request_path: true + + - name: registerDesktopDevicev2 + uris: "{{ api_manager_perfix }}/v2/consumer/desktop_device/credential/register" + upstream_url: "{{ am_util_url }}/v2/consumer/desktop_device/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'desktopAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ x_medium_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateFramework - request_path: "{{ framework_service_prefix }}/v1/update" - upstream_url: "{{ content_service_url }}/v1/framework/update" - strip_request_path: true + + - name: refreshLock + uris: "{{ lock_service_prefix }}/v1/refresh" + upstream_url: "{{ knowledge_mw_service_url }}/v1/lock/refresh" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'contentUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: copyFramework - request_path: "{{ framework_service_prefix }}/v1/copy" - upstream_url: "{{ content_service_url }}/v1/framework/copy" - strip_request_path: true + + - name: refreshToken + uris: "{{ auth_service_prefix }}/v1/refresh/token" + upstream_url: "{{ am_util_url }}/v1/auth/refresh/token" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'refreshTokenCreate' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readFrameworkTerm - request_path: "{{ framework_service_prefix }}/v1/term/read" - upstream_url: "{{ content_service_url }}/v1/framework/term/read" - strip_request_path: true + + - name: registerMobileApp + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_app/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'mobileSuperAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ x_large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchFrameworkTerm - request_path: "{{ framework_service_prefix }}/v1/term/search" - upstream_url: "{{ content_service_url }}/v1/framework/term/search" - strip_request_path: true + + - name: registerMobileAppOpenRAP + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_app_openrap/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app_openrap/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'mobileOpenRAPSuperAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createFrameworkTerm - request_path: "{{ framework_service_prefix }}/v1/term/create" - upstream_url: "{{ content_service_url }}/v1/framework/term/create" - strip_request_path: true + + - name: registerMobileAppTeacherAid + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_app_teacheraid/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app_teacheraid/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'mobileTeacherAidSuperAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateFrameworkTerm - request_path: "{{ framework_service_prefix }}/v1/term/update" - upstream_url: "{{ content_service_url }}/v1/framework/term/update" - strip_request_path: true + + - name: registerMobileDevice + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_device/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'mobileAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readFrameworkCategory - request_path: "{{ framework_service_prefix }}/v1/category/read" - upstream_url: "{{ content_service_url }}/v1/framework/category/read" - strip_request_path: true + + - name: registerMobileDevicev2 + uris: "{{ api_manager_perfix }}/v2/consumer/mobile_device/credential/register" + upstream_url: "{{ am_util_url }}/v2/consumer/mobile_device/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'mobileAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchFrameworkCategory - request_path: "{{ framework_service_prefix }}/v1/category/search" - upstream_url: "{{ content_service_url }}/v1/framework/category/search" - strip_request_path: true + + - name: registerMobileDeviceOpenRAP + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_device_openrap/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device_openrap/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUser + config.whitelist: + - 'mobileOpenRAPAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createFrameworkCategory - request_path: "{{ framework_service_prefix }}/v1/category/create" - upstream_url: "{{ content_service_url }}/v1/framework/category/create" - strip_request_path: true + + - name: registerMobileDeviceTeacherAid + uris: "{{ api_manager_perfix }}/v1/consumer/mobile_device_teacheraid/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device_teacheraid/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'mobileTeacherAidAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateFrameworkCategory - request_path: "{{ framework_service_prefix }}/v1/category/update" - upstream_url: "{{ content_service_url }}/v1/framework/category/update" - strip_request_path: true + + - name: rejectContent + uris: "{{ content_prefix }}/v1/reject" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/reject" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateCourseHierarchy - request_path: '{{ course_service_prefix }}/v1/hierarchy/update' - upstream_url: '{{ content_service_url }}/v1/course/hierarchy/update' - strip_request_path: true + + - name: rejectContentFlag + uris: "{{ content_prefix }}/v1/flag/reject" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/flag/reject" + strip_uri: true plugins: - name: jwt - name: cors - - '{{ statsd_pulgin }}' + - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local - config.hour: '{{ medium_rate_limit_per_hour }}' + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: '{{ small_request_size_limit }}' - - name: submitDatasetRequest - request_path: "{{ data_service_prefix }}/v1/dataset/request/submit" - upstream_url: "{{ content_service_url }}/v1/dataset/request/submit" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: releaseDialcode + uris: "{{ dialcode_service_prefix }}/v1/release" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/release" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listDatasetRequest - request_path: "{{ data_service_prefix }}/v1/dataset/request/list" - upstream_url: "{{ content_service_url }}/v1/dataset/request/list" - strip_request_path: true + + - name: removeMember + uris: "{{ org_service_prefix }}/v1/member/remove" + upstream_url: "{{ learning_service_url }}/v1/org/member/remove" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'orgAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readDatasetRequest - request_path: "{{ data_service_prefix }}/v1/dataset/request/read" - upstream_url: "{{ content_service_url }}/v1/dataset/request/read" - strip_request_path: true + + - name: removeUserFromBatch + uris: "{{ course_service_prefix }}/v1/batch/user/remove" + upstream_url: "{{ lms_service_url }}/v1/course/batch/users/remove" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: channelDatasetRequest - request_path: "{{ data_service_prefix }}/v1/dataset/request" - upstream_url: "{{ content_service_url }}/v1/dataset/request" - strip_request_path: true + + - name: reserveDialcode + uris: "{{ dialcode_service_prefix }}/v1/reserve" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/reserve" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createAssertion - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/create" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/create" - strip_request_path: true + + - name: retireContent + uris: "{{ content_prefix }}/v1/retire" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/retire" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getAssertion - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/read" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/read" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: retireCourse + uris: "{{ course_service_prefix }}/v1/retire" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/retire" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getAssertionList - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/search" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/search" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: retireLock + uris: "{{ lock_service_prefix }}/v1/retire" + upstream_url: "{{ knowledge_mw_service_url }}/v1/lock/retire" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: revokeAssertion - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/delete" + uris: "{{ badge_service_prefix }}/v1/issuer/badge/assertion/delete" upstream_url: "{{ learning_service_url }}/v1/issuer/badge/assertion/delete" - strip_request_path: true + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'badgeAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createBadgeClass - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/create" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/create" - strip_request_path: true + + - name: searchBadgeClass + uris: "{{ badge_service_prefix }}/v1/issuer/badge/search" + upstream_url: "{{ learning_service_url }}/v1/issuer/badge/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'badgeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getBadgeClass - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/read" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/read" - strip_request_path: true + + - name: searchChannel + uris: "{{ channel_service_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/channel/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'channelAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: searchBadgeClass - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/search" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/search" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchContent + uris: "{{ content_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/search" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: publicUser - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: deleteBadgeClass - request_path: "{{ badge_service_prefix }}/v1/issuer/badge/delete" - upstream_url: "{{ learning_service_url }}/v1/issuer/badge/delete" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchCourse + uris: "{{ course_service_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createissuer - request_path: "{{ badge_service_prefix }}/v1/issuer/create" - upstream_url: "{{ learning_service_url }}/v1/issuer/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchData + uris: "{{ data_service_prefix }}/v1/object/search" + upstream_url: "{{ learning_service_url }}/v1/object/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'objectAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getIssuerDetails - request_path: "{{ badge_service_prefix }}/v1/issuer/read" - upstream_url: "{{ learning_service_url }}/v1/issuer/read" - strip_request_path: true + + - name: searchDialcodes + uris: "{{ dialcode_service_prefix }}/v1/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: getAllIssuer - request_path: "{{ badge_service_prefix }}/v1/issuer/list" - upstream_url: "{{ learning_service_url }}/v1/issuer/list" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchFrameworkCategory + uris: "{{ framework_service_prefix }}/v1/category/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/category/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: deleteIssuer - request_path: "{{ badge_service_prefix }}/v1/issuer/delete" - upstream_url: "{{ learning_service_url }}/v1/issuer/delete" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchFrameworkTerm + uris: "{{ framework_service_prefix }}/v1/term/search" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/term/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'frameworkAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createForm - request_path: "{{ data_service_prefix }}/v1/form/create" - upstream_url: "{{ player_service_url }}/plugin/v1/form/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchNotes + uris: "{{ notes_service_prefix }}/v1/search" + upstream_url: "{{ learning_service_url }}/v1/note/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'noteAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateForm - request_path: "{{ data_service_prefix }}/v1/form/update" - upstream_url: "{{ player_service_url }}/plugin/v1/form/update" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchOrgExtended + uris: "{{ org_service_prefix }}/v1/ext/search" + upstream_url: "{{ learning_service_url }}/v1/org/search" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: dataUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: announcementtypeCreate - request_path: "{{ announcement_service_prefix }}/v1/manage/announcement-type/create" - upstream_url: "{{ player_service_url }}/announcement/v1/manage/announcement-type/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: searchOrg + uris: "{{ org_service_prefix }}/v1/search" + upstream_url: "{{ learning_service_url }}/v1/org/search" + strip_uri: true plugins: - - name: jwt - name: cors - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: dataUpdate - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential + config.limit_by: ip - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: announcementtypeUpdate - request_path: "{{ announcement_service_prefix }}/v1/manage/announcement-type/update" - upstream_url: "{{ player_service_url }}/announcement/v1/manage/announcement-type/update" - strip_request_path: true + + - name: searchUser + uris: "{{ user_service_prefix }}/v1/search" + upstream_url: "{{ learning_service_url }}/v1/user/search" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'userTempAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: announcementtypeList - request_path: "{{ announcement_service_prefix }}/v1/manage/announcement-type/list" - upstream_url: "{{ player_service_url }}/announcement/v1/manage/announcement-type/list" - strip_request_path: true + + - name: sendEmailNotification + uris: "{{ user_service_prefix }}/v1/notification/email" + upstream_url: "{{ learning_service_url }}/v1/notification/email" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dataUpdate + config.whitelist: + - 'appUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readForm - request_path: "{{ data_service_prefix }}/v1/form/read" - upstream_url: "{{ player_service_url }}/plugin/v1/form/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: sendNotification + uris: "{{ data_service_prefix }}/v1/notification/send" + upstream_url: "{{ learning_service_url }}/v1/notification/send" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'appAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: masterLocationCreate - request_path: "{{ data_service_prefix }}/v1/location/create" - upstream_url: "{{ learning_service_url }}/v1/location/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: submitContentForReview + uris: "{{ content_prefix }}/v1/review" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/review" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'contentAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: masterLocationUpdate - request_path: "{{ data_service_prefix }}/v1/location/update" - upstream_url: "{{ learning_service_url }}/v1/location/update" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: submitCourseForReview + uris: "{{ course_service_prefix }}/v1/review" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/review" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'courseAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: masterLocationSearch - request_path: "{{ data_service_prefix }}/v1/location/search" - upstream_url: "{{ learning_service_url }}/v1/location/search" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: telemetry + uris: "{{ data_service_prefix }}/v1/telemetry" + upstream_url: "{{ telemetry_service_url }}/v1/telemetry" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'telemetryCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ premium_consumer_large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: masterLocationUpload - request_path: "{{ data_service_prefix }}/v1/bulk/location/upload" - upstream_url: "{{ learning_service_url }}/v1/bulk/location/upload" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: telemetryErrorLogging + uris: "{{ data_service_prefix }}/v1/client/logs" + upstream_url: "{{ analytics_api_service_url }}/data/v1/client/logs" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'telemetryCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: copyContent - request_path: "{{ content_service_prefix }}/v1/copy" - upstream_url: "{{ content_service_url }}/v1/content/copy" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: unblockUser + uris: "{{ user_service_prefix }}/v1/unblock" + upstream_url: "{{ learning_service_url }}/v1/user/unblock" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: fetchURLMetaInformation - request_path: "{{ content_service_prefix }}/v1/fetchmeta" - upstream_url: "{{ content_service_url }}/v1/url/fetchmeta" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateBatch + uris: "{{ course_service_prefix }}/v1/batch/update" + upstream_url: "{{ lms_service_url }}/v1/course/batch/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'courseUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: publishFramework - request_path: "{{ framework_service_prefix }}/v1/publish" - upstream_url: "{{ content_service_url }}/v1/framework/publish" - strip_request_path: true + + - name: updateCertTemplate + uris: "{{ cert_service_prefix }}/v1/template/update" + upstream_url: "{{ cert_service_url }}/cert/v1/template/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: domainUpdate + config.whitelist: + - 'certificateUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: pluginSearch - request_path: "{{ plugin_service_prefix }}/v1/search" - upstream_url: "{{ content_service_url }}/v1/plugins/search" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateChannel + uris: "{{ channel_service_prefix }}/v1/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/channel/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'channelUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: updateCollaborators - request_path: "{{ content_service_prefix }}/v1/collaborator/update" - upstream_url: "{{ content_service_url }}/v1/content/collaborator/update" - strip_request_path: true + uris: "{{ content_prefix }}/v1/collaborator/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/collaborator/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUpdate + config.whitelist: + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: reserveDialcode - request_path: "{{ dialcode_service_prefix }}/v1/reserve" - upstream_url: "{{ content_service_url }}/v1/dialcode/reserve" - strip_request_path: true + + - name: updateContent + uris: "{{ content_prefix }}/v1/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeAdmin + config.whitelist: + - 'contentUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: releaseDialcode - request_path: "{{ dialcode_service_prefix }}/v1/release" - upstream_url: "{{ content_service_url }}/v1/dialcode/release" - strip_request_path: true + + - name: updateContentState + uris: "{{ course_service_prefix }}/v1/content/state/update" + upstream_url: "{{ lms_service_url }}/v1/content/state/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: dialcodeAdmin + config.whitelist: + - 'courseAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createLock - request_path: "{{ lock_service_prefix }}/v1/create" - upstream_url: "{{ content_service_url }}/v1/lock/create" - strip_request_path: true + + - name: updateCourse + uris: "{{ course_service_prefix }}/v1/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'courseUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: refreshLock - request_path: "{{ lock_service_prefix }}/v1/refresh" - upstream_url: "{{ content_service_url }}/v1/lock/refresh" - strip_request_path: true + + - name: updateCourseHierarchy + uris: "{{ course_service_prefix }}/v1/hierarchy/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/course/hierarchy/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'courseUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: retireLock - request_path: "{{ lock_service_prefix }}/v1/retire" - upstream_url: "{{ content_service_url }}/v1/lock/retire" - strip_request_path: true + + - name: updateData + uris: "{{ data_service_prefix }}/v1/object/update" + upstream_url: "{{ learning_service_url }}/v1/object/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'objectUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listLock - request_path: "{{ lock_service_prefix }}/v1/list" - upstream_url: "{{ content_service_url }}/v1/lock/list" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateDesktopApp + uris: "{{ desktop_app_prefix }}/v1/update" + upstream_url: "{{ player_service_url }}/v1/desktop/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'appAccess' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createSession - request_path: "{{ sso_service_prefix }}/v1/create/session" - upstream_url: "{{ player_service_url }}/v1/sso/create/session" - strip_request_path: true + + - name: updateDialcode + uris: "{{ dialcode_service_prefix }}/v1/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'dialcodeUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -3504,294 +3933,143 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - # Data pipeline - - name: "datasetSubmitRequest" - request_path: "{{ data_service_prefix }}/v3/dataset/request/submit" - upstream_url: "{{ sunbird_analytics_api_base_url }}/dataset/request/submit" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "datasetReadRequest" - request_path: "{{ data_service_prefix }}/v3/dataset/request/read" - upstream_url: "{{ sunbird_analytics_api_base_url }}/dataset/request/read" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "datasetListRequest" - request_path: "{{ data_service_prefix }}/v3/dataset/request/list" - upstream_url: "{{ sunbird_analytics_api_base_url }}/dataset/request/list" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "deviceRegister" - request_path: "/v3/device/register" - upstream_url: "{{ sunbird_analytics_api_base_url }}/v1/device/register" - strip_request_path: true - plugins: - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "deviceProfile" - uris: "/v3/device/profile" - upstream_url: "{{ sunbird_analytics_api_base_url }}/v1/device/profile" + - name: updateForm + uris: "{{ data_service_prefix }}/v1/form/update" + upstream_url: "{{ player_service_url }}/plugin/v1/form/update" strip_uri: true - plugins: - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "channelSpecificTelemetryExhaust" - request_path: "{{ data_service_prefix }}/v3/dataset/get" - upstream_url: "{{ sunbird_analytics_api_base_url }}/dataset/get" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - - name: "dataproductsCumulativeMetrics" - request_path: "{{ data_service_prefix }}/v3/metrics" - upstream_url: "{{ sunbird_analytics_api_base_url }}/metrics" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - - name: "experimentcreate" - request_path: "{{ data_service_prefix }}/v1/experiement/create" - upstream_url: "{{ sunbird_analytics_api_base_url }}/experiment/create" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - - name: "experimentread" - request_path: "{{ data_service_prefix }}/v1/experiement/get" - upstream_url: "{{ sunbird_analytics_api_base_url }}/experiment/get" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "configRefresh" - request_path: "{{ config_service_prefix }}/v1/refresh" - upstream_url: "{{ config_service_url }}/config-service/v1/refresh" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'dataAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ small_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - - name: "getConfigByKey" - request_path: "{{ config_service_prefix }}/v1/read" - upstream_url: "{{ config_service_url }}/config-service/v1/read" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ large_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "addSystemSettings" - request_path: "{{ data_service_prefix }}/v1/system/settings/set" - upstream_url: "{{ learning_service_url }}/v1/system/settings/set" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'dataAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "getSystemSettings" - request_path: "{{ data_service_prefix }}/v1/system/settings/get" - upstream_url: "{{ learning_service_url }}/v1/system/settings/get" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "getConfigStatus" - request_path: "{{ config_service_prefix }}/v1/status" - upstream_url: "{{ config_service_url }}/config-service/v1/status" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'dataAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ small_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "getConfigHealth" - request_path: "{{ config_service_prefix }}/v1/health" - upstream_url: "{{ config_service_url }}/config-service/v1/health" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'dataAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ small_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: onboardConsumer - request_path: "{{ api_manager_perfix }}/v1/consumer/create/jwt" - upstream_url: "{{ am_util_url }}/v1/consumer/create/jwt" - strip_request_path: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: kongConsumer + config.whitelist: + - 'formUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ small_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: KongConsumerApi - request_path: "{{ api_manager_perfix }}/v1/consumer" - upstream_url: "{{ am_util_url }}/v1/consumer" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateFramework + uris: "{{ framework_service_prefix }}/v1/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: kongConsumer + config.whitelist: + - 'frameworkUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ small_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: uploadTOC - request_path: "{{ textbook_service_prefix }}/v1/toc/upload" - upstream_url: "{{ lms_service_url }}/v1/textbook/toc/upload" - strip_request_path: true + + - name: updateFrameworkCategory + uris: "{{ framework_service_prefix }}/v1/category/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/category/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'frameworkUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: downloadTOC - request_path: "{{ textbook_service_prefix }}/v1/toc/download" - upstream_url: "{{ lms_service_url }}/v1/textbook/toc/download" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateFrameworkTerm + uris: "{{ framework_service_prefix }}/v1/term/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/framework/term/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentAdmin + config.whitelist: + - 'frameworkUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ large_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: acceptTermsAndCondition - request_path: "{{ user_service_prefix }}/v1/tnc/accept" - upstream_url: "{{ learning_service_url }}/v1/user/tnc/accept" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateLocation + uris: "{{ org_service_prefix }}/v1/location/update" + upstream_url: "{{ learning_service_url }}/v1/notification/location/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'locationUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: generateOtp - request_path: "{{ otp_service_prefix }}/v1/generate" - upstream_url: "{{ learning_service_url }}/v1/otp/generate" - strip_request_path: true + + - name: updateNote + uris: "{{ notes_service_prefix }}/v1/update" + upstream_url: "{{ learning_service_url }}/v1/note/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'noteUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: verifyOtp - request_path: "{{ otp_service_prefix }}/v1/verify" - upstream_url: "{{ learning_service_url }}/v1/otp/verify" - strip_request_path: true + + - name: updateOrg + uris: "{{ org_service_prefix }}/v1/update" + upstream_url: "{{ learning_service_url }}/v1/org/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'orgUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserByKey - request_path: "{{ user_service_prefix }}/v1/get" - upstream_url: "{{ learning_service_url }}/v1/user/get" - strip_request_path: true + + - name: updateOrgStatus + uris: "{{ org_service_prefix }}/v1/status/update" + upstream_url: "{{ learning_service_url }}/v1/org/status/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'orgUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" @@ -3799,917 +4077,1965 @@ kong_apis: - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: registerMobileAppTeacherAid - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_app_teacheraid/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_app_teacheraid/credential/register" - strip_request_path: true + - name: updateOrgType + uris: "{{ org_service_prefix }}/v1/type/update" + upstream_url: "{{ learning_service_url }}/v1/org/type/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: mobileAdminTeacherAid + config.whitelist: + - 'orgUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" + config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: registerMobileDeviceTeacherAid - request_path: "{{ api_manager_perfix }}/v1/consumer/mobile_device_teacheraid/credential/register" - upstream_url: "{{ am_util_url }}/v1/consumer/mobile_device_teacheraid/credential/register" - strip_request_path: true + - name: updatePage + uris: "{{ data_service_prefix }}/v1/page/update" + upstream_url: "{{ lms_service_url }}/v1/page/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: mobileAppTeacherAid + config.whitelist: + - 'pageUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: "contentBadgeLink" - request_path: "{{ badge_service_prefix }}/v1/content/link" - upstream_url: "{{ lms_service_url }}/v1/content/link" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "contentBadgeUnLink" - request_path: "{{ badge_service_prefix }}/v1/content/unlink" - upstream_url: "{{ lms_service_url }}/v1/content/unlink" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "privateUserUpdate" - request_path: "{{ user_service_prefix }}/private/v1/update" - upstream_url: "{{ learning_service_url }}/private/user/v1/update" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "getUserType" - request_path: "{{ user_service_prefix }}/v1/type/list" - upstream_url: "{{ learning_service_url }}/v1/user/type/list" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "masterLocationDelete" - request_path: "{{data_service_prefix }}/v1/location/delete" - upstream_url: "{{ learning_service_url }}/v1/location/delete" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'locationAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "migrateUser" - request_path: "{{ user_service_prefix }}/v1/migrate" - upstream_url: "{{ learning_service_url }}/private/user/v1/migrate" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "privateUserRead" - request_path: "{{ user_service_prefix }}/private/v1/read" - upstream_url: "{{ learning_service_url }}/private/user/v1/read" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "privateUserAssignRole" - request_path: "{{ user_service_prefix }}/private/v1/assign/role" - upstream_url: "{{ learning_service_url }}/private/user/v1/assign/role" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: createUserV3 - request_path: "{{ user_service_prefix }}/v1.3/create" - upstream_url: "{{ user_org_service_url }}/v1.3/user/create" - strip_request_path: true + + - name: updatePageSection + uris: "{{ data_service_prefix }}/v1/page/section/update" + upstream_url: "{{ lms_service_url }}/v1/page/section/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'pageUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: createUserVersion2V3 - request_path: "{{ user_service_prefix }}/v2.3/create" - upstream_url: "{{ user_org_service_url }}/v2.3/user/create" - strip_request_path: true + + - name: updatePublisher + uris: "{{ dialcode_service_prefix }}/v1/publisher/update" + upstream_url: "{{ knowledge_mw_service_url }}/v1/dialcode/publisher/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'dialcodeUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserProfileV3 - request_path: "{{ user_service_prefix }}/v1.3/read" - upstream_url: "{{ user_org_service_url }}/v1.3/user/read" - strip_request_path: true + + - name: updateUser + uris: "{{ user_service_prefix }}/v1/update" + upstream_url: "{{ learning_service_url }}/v1/user/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserProfileV2V3 - request_path: "{{ user_service_prefix }}/v2.3/read" - upstream_url: "{{ user_org_service_url }}/v2.3/user/read" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateUserloginTime + uris: "{{ user_service_prefix }}/v1/update/logintime" + upstream_url: "{{ learning_service_url }}/v1/user/update/logintime" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readUserProfileV3 - request_path: "{{ user_service_prefix }}/v1.3/profile/read" - upstream_url: "{{ user_org_service_url }}/v1.3/user/getuser" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateUserSkill + uris: "{{ user_service_prefix }}/v1/skill/update" + upstream_url: "{{ learning_service_url }}/v1/user/skill/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getUserByKeyV3 - request_path: "{{ user_service_prefix }}/v1.3/get" - upstream_url: "{{ user_org_service_url }}/v1.3/user/get" - strip_request_path: true + + - name: uploadContent + uris: "{{ content_prefix }}/v1/upload" + upstream_url: "{{ knowledge_mw_service_url }}/v1/content/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchUserV3 - request_path: "{{ user_service_prefix }}/v1.3/search" - upstream_url: "{{ user_org_service_url }}/v1.3/user/search" - strip_request_path: true + config.allowed_payload_size: "{{ large_request_size_limit }}" + + - name: uploadCrashLogs + uris: "{{ desktop_app_prefix }}/v1/upload-crash-logs" + upstream_url: "{{ player_service_url }}/v1/desktop/upload-crash-logs" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'telemetryCreate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: getMediaTypesV3 - request_path: "{{ user_service_prefix }}/v1.3/mediatype/list" - upstream_url: "{{ user_org_service_url }}/v1.3/user/mediatype/list" - strip_request_path: true + + - name: uploadTOC + uris: "{{ textbook_service_prefix }}/v1/toc/upload" + upstream_url: "{{ lms_service_url }}/v1/textbook/toc/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'contentCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: profileVisibilityV3 - request_path: "{{ user_service_prefix }}/v1.3/profile/visibility" - upstream_url: "{{ user_org_service_url }}/v1.3/user/profile/visibility" - strip_request_path: true + + - name: userBulkUpload + uris: "{{ user_service_prefix }}/v1/upload" + upstream_url: "{{ learning_service_url }}/v1/user/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: "getUserTypeV3" - request_path: "{{ user_service_prefix }}/v1.3/type/list" - upstream_url: "{{ user_org_service_url }}/v1.3/user/type/list" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: blockUserV3 - request_path: "{{ user_service_prefix }}/v1.3/block" - upstream_url: "{{ user_org_service_url }}/v1.3/user/block" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: UserBulkUploadv2 + uris: "{{ user_service_prefix }}/v2/bulk/upload" + upstream_url: "{{ learning_service_url }}/v2/bulk/user/upload" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'userAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: unblockUserV3 - request_path: "{{ user_service_prefix }}/v1.3/unblock" - upstream_url: "{{ user_org_service_url }}/v1.3/user/unblock" - strip_request_path: true + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: userExistenceApi + uris: "{{ user_service_prefix }}/v1/exists" + upstream_url: "{{ learning_service_url }}/v1/user/exists" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readRoleMappingV3 - request_path: "{{ data_service_prefix }}/v1.3/role/read" - upstream_url: "{{ user_org_service_url }}/v1.3/role/read" - strip_request_path: true + + - name: userExistenceApiV2 + uris: "{{ user_service_prefix }}/v2/exists" + upstream_url: "{{ learning_service_url }}/v2/user/exists" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: assignRoleV3 - request_path: "{{ user_service_prefix }}/v1.3/role/assign" - upstream_url: "{{ user_org_service_url }}/v1.3/user/assign/role" - strip_request_path: true + + - name: userFeed + uris: "{{ user_service_prefix }}/v1/feed" + upstream_url: "{{ learning_service_url }}/v1/user/feed" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: "privateUserAssignRoleV3" - request_path: "{{ user_service_prefix }}/private/v1.3/assign/role" - upstream_url: "{{ user_org_service_url }}/private/user/v1.3/assign/role" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: updateUserV3 - request_path: "{{ user_service_prefix }}/v1.3/update" - upstream_url: "{{ user_org_service_url }}/v1.3/user/update" - strip_request_path: true + + - name: userFeedCreate + uris: "{{ user_service_prefix }}/feed/v1/create" + upstream_url: "{{ learning_service_url }}/v1/user/feed/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: "privateUserUpdateV3" - request_path: "{{ user_service_prefix }}/private/v1.3/update" - upstream_url: "{{ user_org_service_url }}/private/user/v1.3/update" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} - - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: updateUserloginTimeV3 - request_path: "{{ user_service_prefix }}/v1.3/update/logintime" - upstream_url: "{{ user_org_service_url }}/v1.3/user/update/logintime" - strip_request_path: true + + - name: userFeedDelete + uris: "{{ user_service_prefix }}/feed/v1/delete" + upstream_url: "{{ learning_service_url }}/v1/user/feed/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: acceptTermsAndConditionV3 - request_path: "{{ user_service_prefix }}/v1.3/tnc/accept" - upstream_url: "{{ user_org_service_url }}/v1.3/user/tnc/accept" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: userFeedUpdate + uris: "{{ user_service_prefix }}/feed/v1/update" + upstream_url: "{{ learning_service_url }}/v1/user/feed/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: DecryptDataV3 - request_path: "{{ data_service_prefix }}/v1.3/user/decrypt" - upstream_url: "{{ user_org_service_url }}/v1.3/user/data/decrypt" - strip_request_path: true + + - name: userMigrate + uris: "{{ user_service_prefix }}/v1/migrate" + upstream_url: "{{ learning_service_url }}/v1/user/migrate" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: encryptDataV3 - request_path: "{{ data_service_prefix }}/v1.3/user/encrypt" - upstream_url: "{{ user_org_service_url }}/v1.3/user/data/encrypt" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: validateRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/validate" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/validate" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: appUpdate + config.whitelist: + - 'certificateAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createOrgV3 - request_path: "{{ org_service_prefix }}/v1.3/create" - upstream_url: "{{ user_org_service_url }}/v1.3/org/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: verifyOtp + uris: "{{ otp_service_prefix }}/v1/verify" + upstream_url: "{{ learning_service_url }}/v1/otp/verify" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateOrgV3 - request_path: "{{ org_service_prefix }}/v1.3/update" - upstream_url: "{{ user_org_service_url }}/v1.3/org/update" - strip_request_path: true + + - name: verifyRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/verify" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/verify" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'certificateAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: readOrgV3 - request_path: "{{ org_service_prefix }}/v1.3/read" - upstream_url: "{{ user_org_service_url }}/v1.3/org/read" - strip_request_path: true + + - name: registerIgotApp + uris: "{{ api_manager_perfix }}/v1/consumer/igot_app/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/igot_app/credential/register" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUser + config.whitelist: + - 'igotAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: searchOrgV3 - request_path: "{{ org_service_prefix }}/v1.3/search" - upstream_url: "{{ user_org_service_url }}/v1.3/org/search" - strip_request_path: true + + - name: registerIgotDevice + uris: "{{ api_manager_perfix }}/v1/consumer/igot_device/credential/register" + upstream_url: "{{ am_util_url }}/v1/consumer/igot_device/credential/register" + strip_uri: true plugins: + - name: jwt - name: cors - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'igotApp' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: ip + config.hour: "{{ large_rate_limit_per_hour }}" + config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: addMemberV3 - request_path: "{{ org_service_prefix }}/v1.3/member/add" - upstream_url: "{{ user_org_service_url }}/v1.3/org/member/add" - strip_request_path: true + + - name: listReports + uris: "{{ report_service_prefix }}/report/list" + upstream_url: "{{ report_service_url }}/report/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'reportsAdmin' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: removeMemberV3 - request_path: "{{ org_service_prefix }}/v1.3/member/remove" - upstream_url: "{{ user_org_service_url }}/v1.3/org/member/remove" - strip_request_path: true + + - name: getReport + uris: "{{ report_service_prefix }}/report/get" + upstream_url: "{{ report_service_url }}/report/get" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgAdmin + config.whitelist: + - 'reportsAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateOrgStatusV3 - request_path: "{{ org_service_prefix }}/v1.3/status/update" - upstream_url: "{{ user_org_service_url }}/v1.3/org/status/update" - strip_request_path: true + + - name: updateReport + uris: "{{ report_service_prefix }}/report/update" + upstream_url: "{{ report_service_url }}/report/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'reportsUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: listOrgTypesV3 - request_path: "{{ org_service_prefix }}/v1.3/type/list" - upstream_url: "{{ user_org_service_url }}/v1.3/org/type/list" - strip_request_path: true + + - name: deleteReport + uris: "{{ report_service_prefix }}/report/delete" + upstream_url: "{{ report_service_url }}/report/delete" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'reportsAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: createOrgTypeV3 - request_path: "{{ org_service_prefix }}/v1.3/type/create" - upstream_url: "{{ user_org_service_url }}/v1.3/org/type/create" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: createReport + uris: "{{ report_service_prefix }}/report/create" + upstream_url: "{{ report_service_url }}/report/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'reportsCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateOrgTypeV3 - request_path: "{{ org_service_prefix }}/v1.3/type/update" - upstream_url: "{{ user_org_service_url }}/v1.3/org/type/update" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: createReportSummary + uris: "{{ report_service_prefix }}/report/summary/create" + upstream_url: "{{ report_service_url }}/report/summary/create" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: orgUpdate + config.whitelist: + - 'reportsCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: "telemetryErrorLogging" - request_path: "/data/v1/client/logs" - upstream_url: "{{ sunbird_analytics_api_base_url }}/data/v1/client/logs" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: getReportSummary + uris: "{{ report_service_prefix }}/report/summary" + upstream_url: "{{ report_service_url }}/report/summary" + strip_uri: true plugins: - - {name: 'cors'} + - name: jwt + - name: cors - "{{ statsd_pulgin }}" - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: generateCertificate - request_path: "{{ cert_service_prefix }}/v1/certs/generate" - upstream_url: "{{ cert_service_url }}/v1/certs/generate" - strip_request_path: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: orgUpdate - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: "certValidation" - request_path: "{{ user_service_prefix }}/v1/certs/validate" - upstream_url: "{{ learning_service_url }}/private/user/v1/certs/validate" - strip_request_path: true + - name: acl + config.whitelist: + - 'reportsAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listReportSummary + uris: "{{ report_service_prefix }}/report/summary/list" + upstream_url: "{{ report_service_url }}/report/summary/list" + strip_uri: true plugins: - - {name: 'jwt'} - - {name: 'cors'} + - name: jwt + - name: cors - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'userUpdate'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: createCertTemplate - uris: "{{ cert_service_prefix }}/v1/template/create" - upstream_url: "{{ cert_service_url }}/cert/v1/template/create" + - name: acl + config.whitelist: + - 'reportsAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: searchRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/search" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/search" strip_uri: true plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: orgUpdate - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: updateCertTemplate - uris: "{{ cert_service_prefix }}/v1/template/update" - upstream_url: "{{ cert_service_url }}/cert/v1/template/update" + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: readRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/read" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/read" strip_uri: true plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: orgUpdate - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: readCertTemplate - uris: "{{ cert_service_prefix }}/v1/template/read" - upstream_url: "{{ cert_service_url }}/cert/v1/template/read" + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: forgotPassword + uris: "{{ learner_private_route_prefix }}/user/v1/password/reset" + upstream_url: "{{ learning_service_url }}/private/user/v1/password/reset" strip_uri: true plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: orgUpdate - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: "orgAssignKeys" - request_path: "{{ org_service_prefix }}/v1/assign/key" - upstream_url: "{{ learning_service_url }}/v1/org/assign/key" - strip_request_path: true - plugins: - - {name: 'jwt'} - - {name: 'cors'} + - name: jwt + - name: cors - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'orgAdmin'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: "mergeUserAccounts" - request_path: "{{ user_service_prefix }}/v1/account/merge" - upstream_url: "{{ learning_service_url }}/private/user/v1/account/merge" - strip_request_path: true + - name: acl + config.whitelist: + - 'userAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: publishReport + uris: "{{ report_service_prefix }}/report/publish" + upstream_url: "{{ report_service_url }}/report/publish" + strip_uri: true plugins: - - {name: 'jwt'} - - {name: 'cors'} + - name: jwt + - name: cors - "{{ statsd_pulgin }}" - - {name: 'acl', config.whitelist: 'publicUser'} - - {name: 'rate-limiting', config.policy: "local", config.hour: "{{ medium_rate_limit_per_hour }}"} - - {name: 'request-size-limiting', config.allowed_payload_size: "{{ small_request_size_limit }}" } - - name: qrCodeDownload - request_path: "{{ course_service_prefix }}/v1/qrcode/download" - upstream_url: "{{ lms_service_url }}/v1/course/qrcode/download" - strip_request_path: true - plugins: - - name: jwt - - name: cors - - "{{ statsd_pulgin }}" - - name: acl - config.whitelist: courseUser - - name: rate-limiting - config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" - config.limit_by: credential - - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: courseIssueCertificate - request_path: "{{ course_service_prefix }}/batch/cert/v1/issue" - upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/issue" - strip_request_path: true + - name: acl + config.whitelist: + - 'reportsAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: retireReport + uris: "{{ report_service_prefix }}/report/retire" + upstream_url: "{{ report_service_url }}/report/retire" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'reportsAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: courseBatchAddCertificateTemplate - uris: "{{ course_service_prefix }}/batch/cert/v1/template/add" - upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/template/add" + + - name: searchManagedUser + uris: "{{ user_service_prefix }}/v1/managed" + upstream_url: "{{ learning_service_url }}/v1/user/managed" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: courseBatchRemoveCertificateTemplate - uris: "{{ course_service_prefix }}/batch/cert/v1/template/remove" - upstream_url: "{{ lms_service_url }}/v1/course/batch/cert/template/remove" + + - name: createGroup + uris: "{{ group_service_prefix }}/v1/create" + upstream_url: "{{ group_service_url }}/v1/group/create" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: courseUser + config.whitelist: + - 'groupCreate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: downloadCertificate - request_path: "{{ user_service_prefix }}/v1/certs/download" - upstream_url: "{{ learning_service_url }}/v1/user/certs/download" - strip_request_path: true + + - name: updateGroup + uris: "{{ group_service_prefix }}/v1/update" + upstream_url: "{{ group_service_url }}/v1/group/update" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'groupUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: UserBulkUploadv2 - request_path: "{{ user_service_prefix }}/v2/bulk/upload" - upstream_url: "{{ learning_service_url }}/v2/bulk/user/upload" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: listGroup + uris: "{{ group_service_prefix }}/v1/list" + upstream_url: "{{ group_service_url }}/v1/group/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: userAdmin + config.whitelist: + - 'groupAccess' - name: rate-limiting config.policy: local - config.hour: "{{ small_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ medium_request_size_limit }}" - - name: downloadRegCertificate - request_path: "{{ cert_registry_service_prefix }}/v1/certs/download" - upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/download" - strip_request_path: true + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: readGroup + uris: "{{ group_service_prefix }}/v1/read" + upstream_url: "{{ group_service_url }}/v1/group/read" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'groupAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: validateRegCertificate - request_path: "{{ cert_registry_service_prefix }}/v1/certs/validate" - upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/validate" - strip_request_path: true + + - name: deleteGroup + uris: "{{ group_service_prefix }}/v1/delete" + upstream_url: "{{ group_service_url }}/v1/group/delete" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'groupAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateGroupMembership + uris: "{{ group_service_prefix }}/membership/v1/update" + upstream_url: "{{ group_service_url }}/v1/group/membership/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'groupUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: groupActivityAgg + uris: "{{ data_service_prefix }}/v1/group/activity/agg" + upstream_url: "{{ lms_service_url }}/v1/group/activity/agg" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'groupAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: addRegCertificate - request_path: "{{ cert_registry_service_prefix }}/v1/certs/add" - upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/add" - strip_request_path: true + + - name: listForm + uris: "{{ data_service_prefix }}/v1/form/list" + upstream_url: "{{ player_service_url }}/plugin/v1/form/list" + strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'appAdmin' - name: rate-limiting config.policy: local - config.hour: "{{ medium_rate_limit_per_hour }}" + config.hour: "{{ small_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: userMigrate - uris: "{{ user_service_prefix }}/v1/migrate" - upstream_url: "{{ learning_service_url }}/v1/user/migrate" + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: getUserProfileV3 + uris: "{{ user_service_prefix }}/v3/read" + upstream_url: "{{ learning_service_url }}/v3/user/read" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: verifyRegCertificate - uris: "{{ cert_registry_service_prefix }}/v1/certs/verify" - upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/verify" + + - name: updateUserDeclarations + uris: "{{ user_service_prefix }}/v1/declarations" + upstream_url: "{{ learning_service_url }}/v1/user/declarations" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: updateDesktopApp - uris: "{{ desktop_app_prefix }}/v1/update" - upstream_url: "{{ player_service_url }}/v1/desktop/update" + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateUserConsent + uris: "{{ user_service_prefix }}/v1/consent/update" + upstream_url: "{{ learning_service_url }}/v1/user/consent/update" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: contentUser + config.whitelist: + - 'userUpdate' - name: rate-limiting config.policy: local - config.hour: "{{ large_rate_limit_per_hour }}" + config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" - - name: userFeed - uris: "{{ user_service_prefix }}/v1/feed" - upstream_url: "{{ learning_service_url }}/v1/user/feed" + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readUserConsent + uris: "{{ user_service_prefix }}/v1/consent/read" + upstream_url: "{{ learning_service_url }}/v1/user/consent/read" strip_uri: true plugins: - name: jwt - name: cors - "{{ statsd_pulgin }}" - name: acl - config.whitelist: publicUser + config.whitelist: + - 'userAccess' - name: rate-limiting config.policy: local config.hour: "{{ medium_rate_limit_per_hour }}" config.limit_by: credential - name: request-size-limiting - config.allowed_payload_size: "{{ small_request_size_limit }}" + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: "channelSpecificTelemetryExhaust" + uris: "{{ data_service_prefix }}/v3/dataset/get" + upstream_url: "{{ analytics_api_service_url }}/dataset/get" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'dataAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: "submitDataExhaustRequest" + uris: "{{ dataset_service_prefix }}/v1/request/submit" + upstream_url: "{{ analytics_api_service_url }}/request/submit" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'dataCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: "getDataExhaustRequest" + uris: "{{ dataset_service_prefix }}/v1/request/read" + upstream_url: "{{ analytics_api_service_url }}/request/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'dataAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: "listDataExhaustRequest" + uris: "{{ dataset_service_prefix }}/v1/request/list" + upstream_url: "{{ analytics_api_service_url }}/request/list" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'dataAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + - name: createTenantPreferences + uris: "{{ org_service_prefix }}/v2/preferences/create" + upstream_url: "{{ learning_service_url }}/v2/org/preferences/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateTenantPreferences + uris: "{{ org_service_prefix }}/v2/preferences/update" + upstream_url: "{{ learning_service_url }}/v2/org/preferences/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: readTenantPreferencess + uris: "{{ org_service_prefix }}/v2/preferences/read" + upstream_url: "{{ learning_service_url }}/v2/org/preferences/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'orgAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: generateCertificateV2 + uris: "{{ cert_service_prefix }}/v2/certs/generate" + upstream_url: "{{ cert_service_url }}/v2/certs/generate" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: addRegCertificateV2 + uris: "{{ cert_registry_service_prefix }}/v2/certs/add" + upstream_url: "{{ cert_registry_service_url }}/certs/v2/registry/add" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: downloadRegCertificateV2 + uris: "{{ cert_registry_service_prefix }}/v2/certs/download" + upstream_url: "{{ cert_registry_service_url }}/certs/v2/registry/download" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: contentImportAPI + uris: "{{ content_prefix }}/v1/import" + upstream_url: "{{ content_service_url }}/content/v3/import" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: druidGetUpdateListReport + uris: "{{ data_service_prefix }}/v1{{ analytics_report_service_prefix }}" + upstream_url: "{{ analytics_api_service_url }}{{ analytics_report_service_prefix }}" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'druidReportAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: druidDeactivateReport + uris: "{{ data_service_prefix }}/v1{{ analytics_report_service_prefix }}/deactivate" + upstream_url: "{{ analytics_api_service_url }}{{ analytics_report_service_prefix }}/deactivate" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'druidReportAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: druidCreateReport + uris: "{{ data_service_prefix }}/v1{{ analytics_report_service_prefix }}/submit" + upstream_url: "{{ analytics_api_service_url }}{{ analytics_report_service_prefix }}/submit" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'druidReportCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: addRegCertificate + uris: "{{ cert_registry_service_prefix }}/v1/certs/add" + upstream_url: "{{ cert_registry_service_url }}/certs/v1/registry/add" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: generateCertificate + uris: "{{ cert_service_prefix }}/v1/certs/generate" + upstream_url: "{{ cert_service_url }}/v1/certs/generate" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'certificateCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createObjectCategory + uris: "{{ object_category_prefix }}/v1/create" + upstream_url: "{{ taxonomy_service_url }}/object/category/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: readObjectCategory + uris: "{{ object_category_prefix }}/v1/read" + upstream_url: "{{ taxonomy_service_url }}/object/category/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ x_medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateObjectCategory + uris: "{{ object_category_prefix }}/v1/update" + upstream_url: "{{ taxonomy_service_url }}/object/category/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: createObjectCategoryDefinition + uris: "{{ object_category_definition_prefix }}/v1/create" + upstream_url: "{{ taxonomy_service_url }}/object/category/definition/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: readObjectCategoryDefinition + uris: "{{ object_category_definition_prefix }}/v1/read" + upstream_url: "{{ taxonomy_service_url }}/object/category/definition/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ x_medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: updateObjectCategoryDefinition + uris: "{{ object_category_definition_prefix }}/v1/update" + upstream_url: "{{ taxonomy_service_url }}/object/category/definition/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'frameworkUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ small_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: createAsset + uris: "{{ asset_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/asset/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateAsset + uris: "{{ asset_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/asset/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readAsset + uris: "{{ asset_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/asset/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: uploadAsset + uris: "{{ asset_prefix }}/v1/upload" + upstream_url: "{{ content_service_url }}/asset/v4/upload" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: uploadUrlAsset + uris: "{{ asset_prefix }}/v1/upload/url" + upstream_url: "{{ content_service_url }}/asset/v4/upload/url" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: retireAsset + uris: "{{ asset_prefix }}/v1/retire" + upstream_url: "{{ content_service_url }}/asset/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionCreate + uris: "{{ collection_prefix }}/v1/create" + upstream_url: "{{ content_service_url }}/collection/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionRead + uris: "{{ collection_prefix }}/v1/read" + upstream_url: "{{ content_service_url }}/collection/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionUpdate + uris: "{{ collection_prefix }}/v1/update" + upstream_url: "{{ content_service_url }}/collection/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionFlag + uris: "{{ collection_prefix }}/v1/flag" + upstream_url: "{{ content_service_url }}/collection/v4/flag" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionDiscard + uris: "{{ collection_prefix }}/v1/discard" + upstream_url: "{{ content_service_url }}/collection/v4/discard" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionRetire + uris: "{{ collection_prefix }}/v1/retire" + upstream_url: "{{ content_service_url }}/collection/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyAdd + uris: "{{ collection_prefix }}/v1/hierarchy/add" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/add" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyRemove + uris: "{{ collection_prefix }}/v1/hierarchy/remove" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/remove" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyUpdate + uris: "{{ collection_prefix }}/v1/hierarchy/update" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionHierarchyRead + uris: "{{ collection_prefix }}/v1/hierarchy" + upstream_url: "{{ content_service_url }}/collection/v4/hierarchy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + + - name: collectionDialcodeLink + uris: "{{ collection_prefix }}/v1/dialcode/link" + upstream_url: "{{ content_service_url }}/collection/v4/dialcode/link" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: collectionCopy + uris: "{{ collection_prefix }}/v1/copy" + upstream_url: "{{ content_service_url }}/collection/v4/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionCreate + uris: "{{ question_prefix }}/v1/create" + upstream_url: "{{ assessment_service_url }}/question/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionRead + uris: "{{ question_prefix }}/v1/read" + upstream_url: "{{ assessment_service_url }}/question/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionUpdate + uris: "{{ question_prefix }}/v1/update" + upstream_url: "{{ assessment_service_url }}/question/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionRetire + uris: "{{ question_prefix }}/v1/retire" + upstream_url: "{{ assessment_service_url }}/question/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionReview + uris: "{{ question_prefix }}/v1/review" + upstream_url: "{{ assessment_service_url }}/question/v4/review" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionPublish + uris: "{{ question_prefix }}/v1/publish" + upstream_url: "{{ assessment_service_url }}/question/v4/publish" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetCreate + uris: "{{ questionset_prefix }}/v1/create" + upstream_url: "{{ assessment_service_url }}/questionset/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetRead + uris: "{{ questionset_prefix }}/v1/read" + upstream_url: "{{ assessment_service_url }}/questionset/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetUpdate + uris: "{{ questionset_prefix }}/v1/update" + upstream_url: "{{ assessment_service_url }}/questionset/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetRetire + uris: "{{ questionset_prefix }}/v1/retire" + upstream_url: "{{ assessment_service_url }}/questionset/v4/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetReview + uris: "{{ questionset_prefix }}/v1/review" + upstream_url: "{{ assessment_service_url }}/questionset/v4/review" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetPublish + uris: "{{ questionset_prefix }}/v1/publish" + upstream_url: "{{ assessment_service_url }}/questionset/v4/publish" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetUpdateHierarchy + uris: "{{ questionset_prefix }}/v1/hierarchy/update" + upstream_url: "{{ assessment_service_url }}/questionset/v4/hierarchy/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: questionSetReadHierarchy + uris: "{{ questionset_prefix }}/v1/hierarchy" + upstream_url: "{{ assessment_service_url }}/questionset/v4/hierarchy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: createEvent + uris: "{{ event_prefix }}/v4/create" + upstream_url: "{{ content_service_url }}/event/v4/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentCreate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: readEvent + uris: "{{ event_prefix }}/v4/read" + upstream_url: "{{ content_service_url }}/event/v4/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentAccess' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: updateEvent + uris: "{{ event_prefix }}/v4/update" + upstream_url: "{{ content_service_url }}/event/v4/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + + - name: publishEvent + uris: "{{ event_prefix }}/v4/publish" + upstream_url: "{{ content_service_url }}/event/v4/publish" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'courseAdmin' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ small_request_size_limit }}" + + - name: sendEmailNotificationSer + uris: "{{ notification_service_prefix }}/v1/notification/send/sync" + upstream_url: "{{ notification_service_url }}/v1/notification/send/sync" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'appUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" diff --git a/ansible/roles/kong-api/tasks/main.yml b/ansible/roles/kong-api/tasks/main.yml index e38ef6876..365e80c12 100644 --- a/ansible/roles/kong-api/tasks/main.yml +++ b/ansible/roles/kong-api/tasks/main.yml @@ -12,26 +12,9 @@ - name: Copy kong api scripts copy: src=static-files/kong-api-scripts dest=/tmp mode=0755 - #- name: "Ensure kong_admin_api_url is available before running the scripts" - # uri: - # url: "{{ kong_admin_api_url }}/status" - # status_code: 200 - # register: result - # until: result.status == 200 - # retries: 60 - # delay: 5 - -- name: Extracting where the host in which api manager is running - command: bash -lc "sudo docker service ps api-manager_kong | grep Runn | head -n1 | awk '{print $4}'" - register: kong_running_host - delegate_to: "{{groups['swarm-manager-1'][0]}}" - - set_fact: - kong_admin_api_url: "http://{{kong_running_host.stdout}}:8001" - -- name: waiting for 30 seconds for api manager to be fully functional - pause: - seconds: 30 + kong_admin_api_url: "http://{{groups['swarm-manager-1'][0]}}:8001" + when: kong_admin_api_url is not defined - name: Run script to save apis shell: "python /tmp/kong-api-scripts/kong_apis.py /tmp/kong_apis.json --kong-admin-api-url={{ kong_admin_api_url }}" diff --git a/ansible/roles/kong-consumer/defaults/main.yml b/ansible/roles/kong-consumer/defaults/main.yml index 2f2d88b41..2ab952531 100644 --- a/ansible/roles/kong-consumer/defaults/main.yml +++ b/ansible/roles/kong-consumer/defaults/main.yml @@ -2,123 +2,94 @@ kong_admin_api_url: "http://{{kong_host}}:8001" mobile_admin_groups: - - mobileAdmin + - mobileSuperAdmin mobile_app_groups: - - mobileApp + - mobileAdmin mobile_device_groups: - - contentUser - - courseUser - - appUpdate - - configUser - - publicUser + - appAccess + - certificateAccess + - contentAccess + - courseAccess + - deviceCreate + - dialcodeAccess + - locationAccess + - orgAccess + - ssoCreate + - telemetryCreate + - userAccess + - userTempAdmin - userUpdate - - contentUpdate - - domainUser - - dataUpdate - - dialcodeUser - - echoUser + - contentTempAccess + - groupCreate + - groupUpdate + - groupAccess + - groupAdmin + - refreshTokenCreate -integration_partner_groups: - - echoUser - - orgAdmin - - orgUpdate - - orgUser - - userAdmin - - userUpdate - - publicUser - - dataAdmin - - dataUpdate - - domainAdmin - - domainUpdate - - domainUser - - dialcodeAdmin - - dialcodeUpdate - - dialcodeUser +kong_all_consumer_groups: + - announcementAccess + - appAccess - appAdmin - appUpdate - - courseUser - - contentUser + - badgeAccess + - badgeAdmin + - badgeCreate + - badgeUpdate + - certificateAccess + - certificateAdmin + - certificateCreate + - certificateUpdate + - channelAccess + - channelCreate + - channelUpdate + - contentAccess - contentAdmin + - contentCreate - contentUpdate + - courseAccess - courseAdmin + - courseCreate - courseUpdate - - noteAdmin - - noteUpdate - - noteUser - - configUser - -implementaion_user_groups: - - contentAdmin - - contentUpdate - - contentUser - - userAdmin - - userUpdate - - publicUser - - orgAdmin - - orgUpdate - - orgUser - - courseAdmin - - courseUpdate - - courseUser + - deviceCreate + - dialcodeAccess - dialcodeAdmin + - dialcodeCreate - dialcodeUpdate - - dialcodeUser - - privateContentMgmt - -# List all consumer groups in group_vars -kong_all_consumer_groups: - - appAdmin - - appUpdate - - contentAdmin - - contentUpdate - - contentUser - - courseAdmin - - courseUpdate - - courseUser - - echoUser - - mobileAdmin - - mobileApp + - locationAccess + - noteAccess - noteAdmin + - noteCreate - noteUpdate - - noteUser - - orgAdmin - - orgUpdate - - orgUser - - publicUser + - objectAccess + - orgAccess + - orgCreate + - pageAccess + - pluginAccess + - reportsAccess + - reportsAdmin + - reportsCreate + - reportsUpdate + - ssoCreate + - userAccess - userAdmin + - userCreate + - userTempAdmin - userUpdate - - configUser - - dataAdmin - - dataUpdate - - dialcodeAdmin - - dialcodeUpdate - - dialcodeUser - - domainAdmin - - domainUpdate - - domainUser - - privateContentMgmt - -kong_admin_groups: - - kongConsumer + - groupCreate + - groupUpdate + - groupAccess + - groupAdmin + - dataAccess + - druidReportCreate + - druidReportAdmin + - dataCreate -private_groups: - - locationAdmin - -# List the consumers need to be present as state: present -# List the consumers need to be absent as state: absent -# This role will not update / delete consumers not listed here kong_consumers: - - username: api-management-test-user + - username: api-admin groups: "{{ kong_all_consumer_groups }}" state: present - - username: add-any-consumer-to-be-deleted - state: absent - - username: player - groups: "{{ kong_all_consumer_groups }}" - state: present - rate_limits: "{{ premium_consumer_rate_limits }}" - username: mobile_admin groups: "{{ mobile_admin_groups }}" print_credentials: true @@ -129,16 +100,6 @@ kong_consumers: - username: mobile_device groups: "{{ mobile_device_groups }}" state: present - - username: kongAdmin - groups: "{{ kong_admin_groups }}" - state: present - - username: mobile_admin_openrap - groups: ["mobileAdminOpenRAP"] - save_credentials: true - state: present - - username: mobile_app_openrap - groups: ["mobileAppOpenRAP"] - state: present - - username: mobile_device_openrap + - username: mobile_devicev2 groups: "{{ mobile_device_groups }}" state: present diff --git a/ansible/roles/kong-consumer/tasks/main.yml b/ansible/roles/kong-consumer/tasks/main.yml index 155fa8ac1..5c3283603 100644 --- a/ansible/roles/kong-consumer/tasks/main.yml +++ b/ansible/roles/kong-consumer/tasks/main.yml @@ -15,22 +15,18 @@ - name: Copy kong api scripts copy: src=static-files/kong-api-scripts dest=/tmp mode=0755 - #- name: "Ensure kong_admin_api_url is available before running the scripts" - # uri: - # url: "{{ kong_admin_api_url }}/status" - # status_code: 200 - # register: result - # until: result.status == 200 - # retries: 60 - # delay: 5 - -- name: Extracting where the host in which api manager is running - command: bash -lc "sudo docker service ps api-manager_kong | grep Runn | head -n1 | awk '{print $4}'" - register: kong_running_host - delegate_to: "{{groups['swarm-manager-1'][0]}}" +- name: "Ensure kong_admin_api_url is available before running the scripts" + uri: + url: "{{ kong_admin_api_url }}/status" + status_code: 200 + register: result + until: result.status == 200 + retries: 60 + delay: 5 - set_fact: - kong_admin_api_url: "http://{{kong_running_host.stdout}}:8001" + kong_admin_api_url: "http://{{kong_running_host.stdout}}:8001" + when: kong_admin_api_url is not defined - name: Run script to save consumers shell: "python /tmp/kong-api-scripts/kong_consumers.py /tmp/kong_consumers.json --kong-admin-api-url={{ kong_admin_api_url }}" @@ -40,8 +36,8 @@ become: yes file: path={{item}} state=touch with_items: - - /tmp/jwt_token_output.txt - - /tmp/jwt_token.txt + - /tmp/jwt_token_output.txt + - /tmp/jwt_token.txt - name: write to file become: yes diff --git a/ansible/roles/log-es-azure-restore/defaults/main.yml b/ansible/roles/log-es-azure-restore/defaults/main.yml new file mode 100644 index 000000000..2f70c0d69 --- /dev/null +++ b/ansible/roles/log-es-azure-restore/defaults/main.yml @@ -0,0 +1,12 @@ +es_restore_host: localhost +snapshot_base_path: application + +snapshot_create_request_body: { + type: azure, + settings: { + container: "elasticsearch-snapshots", + base_path: "{{ snapshot_base_path }}" + } +} + +#snapshot_base_path: folderNameInTheAzureContainer diff --git a/ansible/roles/log-es-azure-restore/tasks/main.yml b/ansible/roles/log-es-azure-restore/tasks/main.yml new file mode 100644 index 000000000..f45a97587 --- /dev/null +++ b/ansible/roles/log-es-azure-restore/tasks/main.yml @@ -0,0 +1,25 @@ +--- +- name: Set azure snapshot for the first time + uri: + url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup" + method: PUT + body: "{{ snapshot_create_request_body | to_json }}" + headers: + Content-Type: "application/json" + +- name: Restore ES from Azure backup + uri: + url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore" + method: POST + +- name: "Wait for restore to be completed" + uri: + url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_status" + method: GET + return_content: yes + status_code: 200 + body_format: json + register: result + until: result.json.snapshots[0].state == 'SUCCESS' + retries: 1200 + delay: 10 diff --git a/ansible/roles/log-es6/LICENSE b/ansible/roles/log-es6/LICENSE new file mode 100644 index 000000000..0455cacdf --- /dev/null +++ b/ansible/roles/log-es6/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2012-2016 Elasticsearch + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/ansible/roles/log-es6/README.md b/ansible/roles/log-es6/README.md new file mode 100644 index 000000000..298c8f96a --- /dev/null +++ b/ansible/roles/log-es6/README.md @@ -0,0 +1,415 @@ +# ansible-elasticsearch +[![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/) + +**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.** + +Ansible role for 6.x/5.x Elasticsearch. Currently this works on Debian and RedHat based linux systems. Tested platforms are: + +* Ubuntu 14.04/16.04 +* Debian 8 +* Centos 7 + +The latest Elasticsearch versions of 6.x are actively tested. **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.** + +##### Dependency +This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine. + +## Usage + +Create your Ansible playbook with your own tasks, and include the role elasticsearch. You will have to have this repository accessible within the context of playbook, e.g. + +e.g. + +``` +cd /my/repos/ +git clone https://github.com/elastic/ansible-elasticsearch.git +cd /my/ansible/playbook +mkdir -p roles +ln -s /my/repos/ansible-elasticsearch ./roles/elasticsearch +``` + +Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application. This should be unique per node. +The application of the elasticsearch role results in the installation of a node on a host. + +The simplest configuration therefore consists of: + +``` +- name: Simple Example + hosts: localhost + roles: + - { role: elasticsearch, es_instance_name: "node1" } + vars: +``` + +The above installs a single node 'node1' on the hosts 'localhost'. + +This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information. + +### Basic Elasticsearch Configuration + +All Elasticsearch configuration parameters are supported. This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file. +The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters. + +In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation. These can be found in the role's defaults/main.yml file. + +The following illustrates applying configuration parameters to an Elasticsearch instance. By default, Elasticsearch 5.1.2is installed. + +``` +- name: Elasticsearch with custom configuration + hosts: localhost + roles: + #expand to all available parameters + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", + es_config: { + node.name: "node1", + cluster.name: "custom-cluster", + discovery.zen.ping.unicast.hosts: "localhost:9301", + http.port: 9201, + transport.tcp.port: 9301, + node.data: false, + node.master: true, + bootstrap.memory_lock: true, + } + } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + log-es_heap_size: 1g + es_api_port: 9201 +``` + +Whilst the role installs Elasticsearch with the default configuration parameters, the following should be configured to ensure a cluster successfully forms: + +* ```es_config['http.port']``` - the http port for the node +* ```es_config['transport.tcp.port']``` - the transport port for the node +* ```es_config['discovery.zen.ping.unicast.hosts']``` - the unicast discovery list, in the comma separated format ```":,:"``` (typically the clusters dedicated masters) +* ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on. + +The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. + +See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behaviour and available options. +The role makes no attempt to enforce the setting of these are requires users to specify them appropriately. IT is recommended master nodes are listed and thus deployed first where possible. + +A more complex example: + +``` +- name: Elasticsearch with custom configuration + hosts: localhost + roles: + #expand to all available parameters + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", + es_config: { + node.name: "node1", + cluster.name: "custom-cluster", + discovery.zen.ping.unicast.hosts: "localhost:9301", + http.port: 9201, + transport.tcp.port: 9301, + node.data: false, + node.master: true, + bootstrap.memory_lock: true, + } + } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + log-es_heap_size: 1g + es_start_service: false + es_plugins_reinstall: false + es_api_port: 9201 + es_plugins: + - plugin: ingest-geoip + proxy_host: proxy.example.com + proxy_port: 8080 +``` + +#### Important Note + +**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE. These default to "localhost" and 9200 respectively. +If the node is deployed to bind on either a different host or port, these must be changed.** + +### Multi Node Server Installations + +The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. + +An example of a two server deployment is shown below. The first server holds the master and is thus declared first. Whilst not mandatory, this is recommended in any multi node cluster configuration. The second server hosts two data nodes. + +**Note the structure of the below playbook for the data nodes. Whilst a more succinct structures are possible which allow the same role to be applied to a host multiple times, we have found the below structure to be the most reliable with respect to var behaviour. This is the tested approach.** + +``` +- hosts: master_nodes + roles: + - { role: elasticsearch, es_instance_name: "node1", log-es_heap_size: "1g", + es_config: { + cluster.name: "test-cluster", + discovery.zen.ping.unicast.hosts: "elastic02:9300", + http.port: 9200, + transport.tcp.port: 9300, + node.data: false, + node.master: true, + bootstrap.memory_lock: false, + } + } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + ansible_user: ansible + es_plugins: + - plugin: ingest-geoip + + + +- hosts: data_nodes + roles: + - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch", + es_config: { + discovery.zen.ping.unicast.hosts: "elastic02:9300", + http.port: 9200, + transport.tcp.port: 9300, + node.data: true, + node.master: false, + bootstrap.memory_lock: false, + cluster.name: "test-cluster" + } + } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + ansible_user: ansible + es_api_port: 9200 + es_plugins: + - plugin: ingest-geoip + + +- hosts: data_nodes + roles: + - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201, + es_config: { + discovery.zen.ping.unicast.hosts: "elastic02:9300", + http.port: 9201, + transport.tcp.port: 9301, + node.data: true, + node.master: false, + bootstrap.memory_lock: false, + cluster.name: "test-cluster", + } + } + vars: + es_scripts: false + es_templates: false + es_version_lock: false + es_api_port: 9201 + ansible_user: ansible + es_plugins: + - plugin: ingest-geoip + +``` + +Parameters can additionally be assigned to hosts using the inventory file if desired. + +Make sure your hosts are defined in your ```inventory``` file with the appropriate ```ansible_ssh_host```, ```ansible_ssh_user``` and ```ansible_ssh_private_key_file``` values. + +Then run it: + +``` +ansible-playbook -i hosts ./your-playbook.yml +``` + +### Installing X-Pack Features + +X-Pack features, such as Security, are supported. This feature is currently experimental. To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`. + +The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"] + +The following additional parameters allow X-Pack to be configured: + +* ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory. +* ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"``` +* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html) + + +``` +es_role_mapping: + power_user: + - "cn=admins,dc=example,dc=com" + user: + - "cn=users,dc=example,dc=com" + - "cn=admins,dc=example,dc=com" +``` + +* ```es_users``` - Users can be declared here as yml. Two sub keys 'native' and 'file' determine the realm under which realm the user is created. Beneath each of these keys users should be declared as yml entries. e.g. + +``` +es_users: + native: + kibana4_server: + password: changeMe + roles: + - kibana4_server + file: + es_admin: + password: changeMe + roles: + - admin + testUser: + password: changeMeAlso! + roles: + - power_user + - user +``` + + +* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call. Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g. + +``` +es_roles: + file: + admin: + cluster: + - all + indices: + - names: '*' + privileges: + - all + power_user: + cluster: + - monitor + indices: + - names: '*' + privileges: + - all + user: + indices: + - names: '*' + privileges: + - read + kibana4_server: + cluster: + - monitor + indices: + - names: '.kibana' + privileges: + - all + native: + logstash: + cluster: + - manage_index_templates + indices: + - names: 'logstash-*' + privileges: + - write + - delete + - create_index +``` + +* ```es_xpack_license``` - X-Pack license. The license is a json blob. Set the variable directly (possibly protected by Ansible vault) or from a file in the Ansible project on the control machine via a lookup: + +``` +es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}" +``` + +X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter. + +For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml) + +#### Important Note for Native Realm Configuration + +In order for native users and roles to be configured, the role calls the Elasticsearch API. Given security is installed this requires definition of two parameters: + +* ```es_api_basic_auth_username``` - admin username +* ```es_api_basic_auth_password``` - admin password + +These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme). + + +### Additional Configuration + +In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include: + +* ```es_major_version``` Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x". +* ```es_version``` (e.g. "6.1.2"). +* ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost". +* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200** +* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin. +* ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username` +* ```es_start_service``` (true (default) or false) +* ```es_plugins_reinstall``` (true or false (default) ) +* ```es_plugins``` an array of plugin definitions e.g.: +```yml + es_plugins: + - plugin: ingest-geoip +``` +* ```es_allow_downgrades``` For development purposes only. (true or false (default) ) +* ```es_java_install``` If set to false, Java will not be installed. (true (default) or false) +* ```update_java``` Updates Java to the latest version. (true or false (default)) +* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144. +* ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536. +* ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch). +* ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds. + +Earlier examples illustrate the installation of plugins using `es_plugins`. For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version. For community based plugins include the full url. This approach should NOT be used for the X-Pack plugin. See X-Pack below for details here. + +If installing Monitoring or Alerting, ensure the license plugin is also specified. Security configuration is currently not supported but planned for later versions. + +* ```es_user``` - defaults to elasticsearch. +* ```es_group``` - defaults to elasticsearch. +* ```es_user_id``` - default is undefined. +* ```es_group_id``` - default is undefined. + +Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. + +By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories. These directories are created, using the instance and host name, beneath default locations ] +controlled by the following parameters: + +* ```es_pid_dir``` - defaults to "/var/run/elasticsearch". +* ```es_data_dirs``` - defaults to "/var/lib/elasticsearch". This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2" +* ```es_log_dir``` - defaults to "/var/log/elasticsearch". +* ```es_restart_on_change``` - defaults to true. If false, changes will not result in Elasticsearch being restarted. +* ```es_plugins_reinstall``` - defaults to false. If true, all currently installed plugins will be removed from a node. Listed plugins will then be re-installed. + +This role ships with sample scripts and templates located in the [files/scripts/](files/scripts) and [files/templates/](files/templates) directories, respectively. These variables are used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path. +* ```es_scripts_fileglob``` - defaults to `/files/scripts/`. +* ```es_templates_fileglob``` - defaults to `/files/templates/`. + +### Proxy + +To define proxy globaly, set the following variables: + +* ```es_proxy_host``` - global proxy host +* ```es_proxy_port``` - global proxy port + +To define proxy only for a particular plugin during its installation: + +``` + es_plugins: + - plugin: ingest-geoip + proxy_host: proxy.example.com + proxy_port: 8080 +``` + +> For plugins installation, proxy_host and proxy_port are used first if they are defined and fallback to the global proxy settings if not. The same values are currently used for both the http and https proxy settings. + +## Notes + +* The role assumes the user/group exists on the server. The elasticsearch packages create the default elasticsearch user. If this needs to be changed, ensure the user exists. +* The playbook relies on the inventory_name of each host to ensure its directories are unique +* Changing an instance_name for a role application will result in the installation of a new component. The previous component will remain. +* KitchenCI has been used for testing. This is used to confirm images reach the correct state after a play is first applied. We currently test only the latest version of 6.x on +all supported platforms. +* The role aims to be idempotent. Running the role multiple times, with no changes, should result in no state change on the server. If the configuration is changed, these will be applied and +Elasticsearch restarted where required. +* Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7. All other versions use init for service scripts. +* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests. + +## IMPORTANT NOTES RE PLUGIN MANAGEMENT + +* If the ES version is changed, all plugins will be removed. Those listed in the playbook will be re-installed. This is behaviour is required in ES 6.x. +* If no plugins are listed in the playbook for a node, all currently installed plugins will be removed. +* The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed. Should users wish to re-install plugins they should set es_plugins_reinstall to true. This will cause all currently installed plugins to be removed and those listed to be installed. + +## Questions on Usage + +We welcome questions on how to use the role. However, in order to keep the github issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch. This is monitored by the maintainers. diff --git a/ansible/roles/log-es6/ansible.cfg b/ansible/roles/log-es6/ansible.cfg new file mode 100644 index 000000000..d9a8c5019 --- /dev/null +++ b/ansible/roles/log-es6/ansible.cfg @@ -0,0 +1 @@ +[defaults] \ No newline at end of file diff --git a/ansible/roles/log-es6/defaults/main.yml b/ansible/roles/log-es6/defaults/main.yml new file mode 100644 index 000000000..13f752415 --- /dev/null +++ b/ansible/roles/log-es6/defaults/main.yml @@ -0,0 +1,44 @@ +--- +es_major_version: "6.x" +es_version: "6.8.6" +es_version_lock: false +es_use_repository: true +es_templates_fileglob: "files/templates/*.json" +es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch" +es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main" +es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main" +es_start_service: true +es_java_install: true +update_java: false +es_restart_on_change: true +es_plugins_reinstall: false +es_scripts: false +es_templates: true +es_user: elasticsearch +es_group: elasticsearch +es_config_log4j2: log4j2.properties.j2 +#Need to provide default directories +es_pid_dir: "/var/run/elasticsearch" +es_data_dirs: "/var/lib/elasticsearch" +es_log_dir: "/var/log/elasticsearch" +es_max_open_files: 65536 +es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}" +es_max_map_count: 262144 +es_allow_downgrades: false +es_enable_xpack: false +es_xpack_features: ["alerting","monitoring","graph","ml"] +#These are used for internal operations performed by ansible. +#They do not affect the current configuration +es_api_host: "localhost" +es_api_port: 9200 +es_debian_startup_timeout: 10 +env_dir: /etc/default +log_es_heap_size: "2g" +# Since ansible 2.2 the following variables need to be defined +# to allow the role to be conditionally played with a when condition. +pid_dir: '' +log_dir: '' +conf_dir: '' +data_dirs: '' +# JVM custom parameters +es_jvm_custom_parameters: '' diff --git a/ansible/roles/log-es6/files/logging/log4j2.properties.custom.j2 b/ansible/roles/log-es6/files/logging/log4j2.properties.custom.j2 new file mode 100644 index 000000000..9a2a60f88 --- /dev/null +++ b/ansible/roles/log-es6/files/logging/log4j2.properties.custom.j2 @@ -0,0 +1,76 @@ +#CUSTOM LOG4J FILE + +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = info + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs}.log +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true + +rootLogger.level = debug +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 10mb +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = debug +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log +appender.index_search_slowlog_rolling.layout.type = PatternLayout +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval = 1 +appender.index_search_slowlog_rolling.policies.time.modulate = true + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = debug +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval = 1 +appender.index_indexing_slowlog_rolling.policies.time.modulate = true + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = debug +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/ansible/roles/log-es6/files/mappings/learning.json b/ansible/roles/log-es6/files/mappings/learning.json new file mode 100644 index 000000000..c5991a3d5 --- /dev/null +++ b/ansible/roles/log-es6/files/mappings/learning.json @@ -0,0 +1,783 @@ +{ + "index_patterns": "learning-*", + "settings": { + "number_of_shards": 5 + }, + "mappings": { + "events_v1": { + "dynamic": false, + "properties": { + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "uid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "context":{ + "properties": { + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "model": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "granularity": { + "type": "keyword" + }, + "date_range": { + "properties": { + "from": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "to": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + } + } + }, + "dimensions": { + "properties": { + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "gdata": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "sid": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "group_user": { + "type": "boolean" + }, + "anonymous_user": { + "type": "boolean" + }, + "uid": { + "type": "keyword" + }, + "ss_mid": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + }, + "period": { + "type": "integer" + }, + "content_id": { + "type": "keyword" + }, + "item_id": { + "type": "keyword" + }, + "author_id": { + "type": "keyword" + }, + "loc": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "context_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "object_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + } + } + }, + "edata": { + "properties": { + "eks": { + "properties": { + "total_users_count": { + "type": "long" + }, + "total_devices_count": { + "type": "long" + }, + "total_content_count": { + "type": "long" + }, + "content_ids": { + "type": "keyword" + }, + "device_ids": { + "type": "keyword" + }, + "user_ids": { + "type": "keyword" + }, + "contents": { + "type": "keyword" + }, + "unique_users": { + "type": "keyword" + }, + "avg_ts_session": { + "type": "double" + }, + "total_sessions": { + "type": "long" + }, + "avg_interactions_min": { + "type": "double" + }, + "total_interactions": { + "type": "long" + }, + "avg_pageviews": { + "type": "double" + }, + "total_ts": { + "type": "double" + }, + "time_diff": { + "type": "double" + }, + "time_spent": { + "type": "double" + }, + "interact_events_per_min": { + "type": "double" + }, + "interact_events_count": { + "type": "long" + }, + "total_pageviews_count": { + "type": "long" + }, + "page_views_count": { + "type": "long" + }, + "ce_visits": { + "type": "long" + }, + "start_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "end_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncDate": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "timeSpent": { + "type": "double" + }, + "interactEventsPerMin": { + "type": "double" + }, + "mimeType": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "timeDiff": { + "type": "double" + }, + "mode": { + "type": "keyword" + }, + "telemetryVersion": { + "type": "keyword" + }, + "noOfInteractEvents": { + "type": "long" + }, + "interruptTime": { + "type": "double" + }, + "contentCount": { + "type": "long" + }, + "content": { + "type": "keyword" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "first_visit": { + "type": "boolean" + }, + "noOfAttempts": { + "type": "integer" + }, + "m_side_loads": { + "type": "integer" + }, + "m_downloads": { + "type": "integer" + }, + "m_avg_rating": { + "type": "double" + }, + "m_ratings": { + "type": "nested", + "properties": { + "rating": { + "type": "integer" + }, + "time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + }, + "m_comments": { + "type": "object" + }, + "ce_total_ts": { + "type": "double" + }, + "ce_percent_sessions": { + "type": "double" + }, + "ce_visits_count": { + "type": "long" + }, + "ce_total_visits": { + "type": "long" + }, + "ce_percent_ts": { + "type": "double" + }, + "avg_ts": { + "type": "double" + }, + "inc_res_count": { + "type": "long" + }, + "incorrect_res": { + "type": "nested", + "properties": { + "resp": { + "type": "keyword" + }, + "mmc": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "total_count": { + "type": "long" + }, + "correct_res_count": { + "type": "long" + }, + "correct_res": { + "type": "keyword" + }, + "itemId": { + "type": "keyword" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + }, + "new_user_count": { + "type": "long" + }, + "anon_total_sessions": { + "type": "long" + }, + "anon_total_ts": { + "type": "double" + }, + "anon_avg_ts_session": { + "type": "double" + }, + "percent_new_users_count": { + "type": "double" + }, + "ce_total_sessions": { + "type": "long" + }, + "unique_users_count": { + "type": "long" + }, + "stageVisitCount": { + "type": "long" + }, + "interactEventsCount": { + "type": "long" + }, + "interactEvents": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + } + } + }, + "stageId": { + "type": "keyword" + }, + "visitCount": { + "type": "long" + }, + "num_downloads": { + "type": "long" + }, + "num_sideloads": { + "type": "long" + }, + "avg_depth": { + "type": "double" + }, + "publish_pipeline_summary": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "events_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "eventsSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "screenSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "visitCount": { + "type": "long" + } + } + }, + "activitySummary": { + "type": "nested", + "properties": { + "actType": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "page_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "visit_count": { + "type": "long" + } + } + }, + "env_summary": { + "type": "nested", + "properties": { + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "itemResponses": { + "type": "nested", + "properties": { + "itemId": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + } + } + } + } + } + } + }, + "etags": { + "properties": { + "app": { + "type": "keyword" + }, + "partner": { + "type": "keyword" + }, + "dims": { + "type": "keyword" + } + } + }, + "geoip": { + "properties": { + "location": { + "type": "geo_point" + } + }, + "type": "object" + }, + "contentdata": { + "properties": { + "ageGroup": { + "type": "keyword" + }, + "author": { + "type": "keyword" + }, + "audience": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "collaborators": { + "type": "keyword" + }, + "collections": { + "type": "keyword" + }, + "concepts": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "curriculum": { + "type": "keyword" + }, + "developer": { + "type": "keyword" + }, + "domain": { + "type": "keyword" + }, + "downloadUrl": { + "type": "keyword" + }, + "downloads": { + "type": "long" + }, + "edition": { + "type": "keyword" + }, + "genre": { + "type": "keyword" + }, + "gradeLevel": { + "type": "keyword" + }, + "keywords": { + "type": "keyword" + }, + "me_totalDevices": { + "type": "long" + }, + "me_totalDownloads": { + "type": "long" + }, + "me_totalInteractions": { + "type": "long" + }, + "me_totalRatings": { + "type": "long" + }, + "me_totalSessionsCount": { + "type": "long" + }, + "me_totalSideloads": { + "type": "long" + }, + "me_totalTimespent": { + "type": "long" + }, + "me_totalUsage": { + "type": "long" + }, + "medium": { + "type": "keyword" + }, + "methods": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + }, + "popularity": { + "type": "long" + }, + "portalOwner": { + "type": "keyword" + }, + "publication": { + "type": "keyword" + }, + "publisher": { + "type": "keyword" + }, + "rating": { + "type": "long" + }, + "size": { + "type": "long" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "subject": { + "type": "keyword" + }, + "templateType": { + "type": "keyword" + }, + "theme": { + "type": "keyword" + }, + "words": { + "type": "keyword" + } + } + }, + "itemdata": { + "properties": { + "concepts": { + "type": "keyword" + }, + "createdBy": { + "type": "keyword" + }, + "createdOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "lastUpdatedBy": { + "type": "keyword" + }, + "lastUpdatedOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "media": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "num_answers": { + "type": "long" + }, + "owner": { + "type": "keyword" + }, + "qlevel": { + "type": "keyword" + }, + "question": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "template": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "@version": { + "type": "keyword" + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/scripts/calculate-score.groovy b/ansible/roles/log-es6/files/scripts/calculate-score.groovy new file mode 100644 index 000000000..442c25ca6 --- /dev/null +++ b/ansible/roles/log-es6/files/scripts/calculate-score.groovy @@ -0,0 +1 @@ +log(_score * 2) + my_modifier \ No newline at end of file diff --git a/ansible/roles/log-es6/files/system_key b/ansible/roles/log-es6/files/system_key new file mode 100644 index 000000000..91962910d Binary files /dev/null and b/ansible/roles/log-es6/files/system_key differ diff --git a/ansible/roles/log-es6/files/templates/backend.json b/ansible/roles/log-es6/files/templates/backend.json new file mode 100644 index 000000000..07161ca34 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/backend.json @@ -0,0 +1,411 @@ +{ + "index_patterns" : "backend-*", + "settings" : { + "number_of_shards" : 5 + }, + "mappings" : { + "events" : { + "dynamic": false, + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "@version": { + "type": "keyword" + }, + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "actor": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "edata": { + "properties": { + "comments": { + "type": "keyword" + }, + "correlationid": { + "type": "keyword" + }, + "duration": { + "type": "double" + }, + "data": { + "type": "keyword" + }, + "dir": { + "type": "keyword" + }, + "errtype": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "level": { + "type": "keyword" + }, + "loc": { + "type": "keyword" + }, + "message": { + "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "pass": { + "type": "keyword" + }, + "prevstate": { + "type": "keyword" + }, + "pageid": { + "type": "keyword" + }, + "query": { + "type": "keyword" + }, + "rating": { + "type": "double" + }, + "score": { + "type": "double" + }, + "size": { + "type": "double" + }, + "state": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + }, + "items": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "origin": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + } + } + }, + "item": { + "properties": { + "id": { + "type": "keyword" + }, + "maxscore": { + "type": "long" + }, + "exlength": { + "type": "long" + }, + "uri": { + "type": "keyword" + }, + "desc": { + "type": "keyword" + }, + "title": { + "type": "keyword" + } + } + }, + "target": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "category": { + "type": "keyword" + }, + "parent": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + } + } + }, + "visits": { + "type": "nested", + "properties": { + "objid": { + "type": "keyword" + }, + "objtype": { + "type": "keyword" + } + } + }, + "plugin": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "category": { + "type": "keyword" + } + } + }, + "object": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "uaspec": { + "properties": { + "agent": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "system": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "raw": { + "type": "keyword" + } + } + }, + "dspec": { + "properties": { + "camera": { + "type": "keyword" + }, + "cpu": { + "type": "keyword" + }, + "edisk": { + "type": "double" + }, + "id": { + "type": "keyword" + }, + "idisk": { + "type": "double" + }, + "make": { + "type": "keyword" + }, + "mem": { + "type": "double" + }, + "os": { + "type": "keyword" + }, + "scrn": { + "type": "double" + }, + "sims": { + "type": "double" + } + } + } + } + }, + "context": { + "properties": { + "channel": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "sid": { + "type": "keyword" + }, + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "cdata": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + } + } + }, + "object": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "parentid": { + "type": "keyword" + }, + "parenttype": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + } + } + }, + "metadata": { + "properties": { + "source": { + "type": "keyword" + }, + "index_name": { + "type": "keyword" + }, + "index_type": { + "type": "keyword" + }, + "source_eid": { + "type": "keyword" + }, + "source_mid": { + "type": "keyword" + }, + "pump": { + "type": "keyword" + } + } + }, + "flags": { + "properties": { + "v2_converted": { + "type": "boolean" + }, + "dd_processed": { + "type": "boolean" + }, + "tv_processed": { + "type": "boolean" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/cbatch-assessment.json b/ansible/roles/log-es6/files/templates/cbatch-assessment.json new file mode 100644 index 000000000..0dd91aa4a --- /dev/null +++ b/ansible/roles/log-es6/files/templates/cbatch-assessment.json @@ -0,0 +1,269 @@ +{ + "index_patterns": "cbatch-assessment-*", + "settings": { + "index": { + "number_of_shards": "5", + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "ngram", + "max_gram": "20" + } + }, + "analyzer": { + "cs_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "type": "custom", + "tokenizer": "keyword" + }, + "cs_search_analyzer": { + "filter": [ + "lowercase", + "standard" + ], + "type": "custom", + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "1" + }, + "blocks": { + "read_only_allow_delete": "false" + } + }, + "mappings": { + "_doc": { + "dynamic": "false", + "properties": { + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "batchId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "userId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "courseId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "contentName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "externalId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "districtName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "maskedEmail": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "maskedPhone": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "userName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "rootOrgName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "subOrgName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "score": { + "type": "text", + "fields": { + "raw": { + "type": "text" + } + } + }, + "totalScore": { + "type": "text", + "fields": { + "raw": { + "type": "text" + } + } + }, + "reportUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/cbatchstatsindex.json b/ansible/roles/log-es6/files/templates/cbatchstatsindex.json new file mode 100644 index 000000000..0af8ac7be --- /dev/null +++ b/ansible/roles/log-es6/files/templates/cbatchstatsindex.json @@ -0,0 +1,309 @@ +{ + "index_patterns": "cbatchstats-*", + "settings": { + "index": { + "number_of_shards": "5", + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "ngram", + "max_gram": "20" + } + }, + "analyzer": { + "cs_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "type": "custom", + "tokenizer": "keyword" + }, + "cs_search_analyzer": { + "filter": [ + "lowercase", + "standard" + ], + "type": "custom", + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "1" + }, + "blocks": { + "read_only_allow_delete": "false" + } + }, + "mappings": { + "_doc": { + "dynamic": "false", + "properties": { + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "batchId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "completedPercent": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "courseId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "districtName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "enrolledOn": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "lastUpdatedOn": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "maskedEmail": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "maskedPhone": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "rootOrgName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "subOrgName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "userId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "completedOn": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "blockName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + }, + "externalId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower", + "fielddata": true + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fielddata": true + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/compositeindex.json b/ansible/roles/log-es6/files/templates/compositeindex.json new file mode 100644 index 000000000..d19a18994 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/compositeindex.json @@ -0,0 +1,5122 @@ +{ + "index_patterns": "compositesearch", + "settings": { + "index": { + "max_ngram_diff" : "29", + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "nGram", + "max_gram": "30" + } + }, + "analyzer": { + "cs_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "tokenizer": "keyword" + }, + "cs_search_analyzer": { + "filter": [ + "standard", + "lowercase" + ], + "type": "custom", + "tokenizer": "standard" + } + } + } + }, + "number_of_shards": 5 + }, + "mappings": { + "cs": { + "dynamic_templates": [ + { + "longs": { + "mapping": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "match_mapping_type": "long" + } + }, + { + "booleans": { + "mapping": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "match_mapping_type": "boolean" + } + }, + { + "doubles": { + "mapping": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "match_mapping_type": "double" + } + }, + { + "dates": { + "mapping": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "match_mapping_type": "date" + } + }, + { + "strings": { + "mapping": { + "type": "text", + "copy_to": "all_fields", + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + } + }, + "match_mapping_type": "string" + } + }, + { + "nested": { + "mapping": { + "type": "nested", + "fields": { + "type": "nested" + } + }, + "match_mapping_type": "object" + } + } + ], + "properties": { + "Audio": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_FUNC_OBJECT_TYPE": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_INDEXABLE_METADATA_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_IN_RELATIONS_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_NON_INDEXABLE_METADATA_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_OUT_RELATIONS_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_REQUIRED_PROPERTIES": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_SYSTEM_TAGS_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_SYS_NODE_TYPE": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "IL_UNIQUE_ID": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "Link": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "RhymingSound": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "SET_OBJECT_TYPE_KEY": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "SET_TYPE": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "SYS_INTERNAL_LAST_UPDATED_ON": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "TemplateType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "actions": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "activityType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "activity_class": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ageGroup": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "allowupdate_status": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "altIsoSymbol": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "answer": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "antonyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "apiId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "appIcon": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "appIconLabel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "appId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "applicability": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "artifactUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "attribution": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "attributions": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "audience": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "author": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "authoringScore": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "averageComplexity": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "avgGamingTime": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "badgeAssertions": { + "properties": { + "assertionId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "badgeClassId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "badgeClassName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "createdTS": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "issuerId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + } + } + }, + "baseUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "bloomsTaxonomyLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "board": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "body": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "c_null_open_batch_count": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "c_null_private_batch_count": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "c_test_name_open_batch_count": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "cases": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "categories": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "category": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "categoryinstances": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "channel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "channels": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "chapterName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "chapterNumber": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "childNodes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "children": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "class": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "code": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "collaborators": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "collections": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "communication_scheme": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "compatibilityLevel": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "conceptData": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "conceptIds": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "concepts": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "consumerGroups": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "consumerId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentDisposition": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentEncoding": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentFilter": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentMeta": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contentTypesCount": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "contents": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "converse": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "copyType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "copyright": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "cost": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "count_grade_1": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_grade_2": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_grade_3": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_CC": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_DEM": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_ECH": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_INJ": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_INTF": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_JJ": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_NEG": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_NN": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_NST": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_PRP": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_PSP": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_QC": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_QF": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_QO": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_RB": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_RDP": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_RP": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_VAUX": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_VM": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_pos_WQ": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_source_Karnataka_Govt_Textbooks": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_source_Rajasthan_Textbooks": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "count_textbooks": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "createdBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "createdFor": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "createdOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "creator": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "curriculum": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "defaultFramework": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "defaultRes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "description": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "developer": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "dialcodes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "dimensions": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "domain": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "downloadUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "duration": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "edition": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "editorState": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ekstepWordnet": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "endPoint": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "endsWithAkshara": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "example": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "exampleSentences": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "externalValidation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "faculty": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "failPopup": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "feedback": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "filter": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "flagReasons": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "flaggedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "flags": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "follows": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "format": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "foundation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "framework": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "frameworkId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "frameworks": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "fw_hierarchy": { + "type": "text", + "index": false + }, + "gender": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "genders": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "genieScore": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "genre": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "gloss": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "grade": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "gradeLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "graph_id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "grayScaleAppIcon": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "hasAntonyms": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "hasConnotative": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "hasSynonyms": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "hints": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "holonyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "hypernyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "hyponyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "i18n": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "idealScreenDensity": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "idealScreenSize": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "illustrator": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "illustrators": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "image": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "imageCredits": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "index": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "indowordnetId": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "inflections": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "instruction": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "instructions": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "interactivityLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "internalValidation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ipaSymbol": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "isConnotative": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "isLoanWord": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "isPhrase": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "isoCode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "isoSymbol": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "itemType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "item_sets": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "items": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "keywords": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "keywords_bkp": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "label": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "langid": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "language": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "languageCode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "languageId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "languageLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "lastFlaggedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "lastPublishDate": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "lastPublishedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "lastPublishedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "lastSubmittedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "lastUpdatedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "lastUpdatedOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "launchUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "leafNodesCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "learnerLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "learningObjective": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "lemma": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "level": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "lhs_options": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "license": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "limit": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "liveWords": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "material": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "maxChainLength": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "maxWords": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "max_score": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_audiosCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_averageInteractionsPerMin": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_averageRating": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_averageSessionsPerDevice": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_averageTimespentPerSession": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_avgCreationTsPerSession": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_creationSessions": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_creationTimespent": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_imagesCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_timespentDraft": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_timespentReview": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_totalComments": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalDevices": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalDownloads": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalInteractions": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalRatings": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalSessionsCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalSideloads": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "me_totalTimespent": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "me_videosCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "meaning": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "media": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "mediaType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "mediatype": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "medium": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "meronyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "method": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "methods": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "mimeType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "mimeTypesCount": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "minChainLength": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "model": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "model_sample": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "morphology": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "nodeType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "node_id": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "notes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "num_answers": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "objects": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "objectsUsed": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "occurrenceCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "oldContentBody": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "optStatus": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "options": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "organization": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "origin": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "orthographic_complexity": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "os": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "osId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "owner": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pageNumber": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "parent": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "partial_scoring": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "person": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "phonologic_complexity": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "pictures": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pkgVersion": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "plurality": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "popularity": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "portalOwner": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pos": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "posTags": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pos_categories": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "posterImage": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pragma": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "prevState": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "previewUrl": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "primaryMeaning": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "primaryMeaningId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "product": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "pronunciations": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "publication": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "publishError": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "publish_type": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "publisher": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "purpose": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "qid": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "qlevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "qtype": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "question": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "questionImage": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "question_audio": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "question_count": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "question_image": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "references": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "renderingHints": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "resourceType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "resources": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "responses": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "rhs_options": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "rownum": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "ruleId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ruleObjectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ruleScript": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "s3Key": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "sYS_INTERNAL_LAST_UPDATED_ON": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "sampleUsages": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "savingPopup": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "screenshots": { + "type": "text", + "index": false + }, + "semanticVersion": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "showNotification": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "shuffle": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "size": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "skills": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "softConstraints": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "soundCredits": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "source": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "sourceTypes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "sources": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "startWordsSize": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "startsWithAkshara": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "state": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "strict_sequencing": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "subject": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "sublevel": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "successPopup": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "syllableCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "syllableNotation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "syllables": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "synonyms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "synsetCount": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "synsets": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "tags": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "tempData": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "template": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "templateName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "templateType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "template_id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "terms": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "text": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "textbookName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "theme": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "themes": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "thumbnail": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "title": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "toc_url": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "tools": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "totalGamingTime": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "total_items": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "translations": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "ttl": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "tutor": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "type": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "unicode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "unicodeNotation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "usedByContent": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "used_for": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "usesContent": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "variants": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "varna": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "version": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "versionCheckMode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "versionKey": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "visibility": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "voiceCredits": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "vowel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "weightages": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "wordImages": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "wordListIds": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "wordLists": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "word_complexity": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "words": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "wordsets": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "workers": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + }, + "year": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "cs_index_analyzer", + "search_analyzer": "cs_search_analyzer" + } + } + } + } +} diff --git a/ansible/roles/log-es6/files/templates/dialcode.json b/ansible/roles/log-es6/files/templates/dialcode.json new file mode 100644 index 000000000..12e600ad9 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/dialcode.json @@ -0,0 +1,269 @@ +{ + "index_patterns": "dialcode", + "settings": { + "index": { + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "nGram", + "max_gram": "20" + } + }, + "analyzer": { + "dc_search_analyzer": { + "filter": [ + "standard", + "lowercase" + ], + "type": "custom", + "tokenizer": "standard" + }, + "dc_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "tokenizer": "keyword" + } + } + } + }, + "number_of_shards": 5 + }, + "mappings": { + "dc": { + "dynamic_templates": [ + { + "longs": { + "mapping": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "match_mapping_type": "long" + } + }, + { + "booleans": { + "mapping": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "match_mapping_type": "boolean" + } + }, + { + "doubles": { + "mapping": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "match_mapping_type": "double" + } + }, + { + "dates": { + "mapping": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "match_mapping_type": "date" + } + }, + { + "strings": { + "mapping": { + "type": "text", + "copy_to": "all_fields", + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + } + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "batchCode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "batchcode": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "channel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "dialcode_index": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "generated_on": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "published_on": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "publisher": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + }, + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "dc_index_analyzer", + "search_analyzer": "dc_search_analyzer" + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/failed-telemetry.json b/ansible/roles/log-es6/files/templates/failed-telemetry.json new file mode 100644 index 000000000..5d474e723 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/failed-telemetry.json @@ -0,0 +1,106 @@ +{ + "index_patterns" : "failed-telemetry-*", + "settings" : { + "number_of_shards" : 5 + }, + "mappings" : { + "events" : { + "dynamic": false, + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "@version": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "metadata": { + "properties": { + "source": { + "type": "keyword" + }, + "index_name": { + "type": "keyword" + }, + "index_type": { + "type": "keyword" + } + } + }, + "context": { + "properties": { + "channel": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "sid": { + "type": "keyword" + }, + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "cdata": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/lp_audit_log.json b/ansible/roles/log-es6/files/templates/lp_audit_log.json new file mode 100644 index 000000000..b33cf788b --- /dev/null +++ b/ansible/roles/log-es6/files/templates/lp_audit_log.json @@ -0,0 +1,314 @@ +{ + "index_patterns": "lp_audit_log", + "settings": { + "index": { + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "nGram", + "max_gram": "20" + } + }, + "analyzer": { + "ah_search_analyzer": { + "filter": [ + "standard", + "lowercase" + ], + "type": "custom", + "tokenizer": "standard" + }, + "keylower": { + "filter": "lowercase", + "tokenizer": "keyword" + }, + "ah_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + } + } + } + }, + "number_of_shards": 5 + }, + "mappings": { + "ah": { + "dynamic_templates": [ + { + "longs": { + "mapping": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "match_mapping_type": "long" + } + }, + { + "booleans": { + "mapping": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "match_mapping_type": "boolean" + } + }, + { + "doubles": { + "mapping": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "match_mapping_type": "double" + } + }, + { + "dates": { + "mapping": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "match_mapping_type": "date" + } + }, + { + "strings": { + "mapping": { + "type": "text", + "copy_to": "all_fields", + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + } + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "@version": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "audit_id": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "createdOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "graphId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "label": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "logRecord": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "objectId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "operation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "requestId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "summary": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + }, + "userId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "ah_index_analyzer", + "search_analyzer": "ah_search_analyzer" + } + } + } + } +} diff --git a/ansible/roles/log-es6/files/templates/suggestionindex.json b/ansible/roles/log-es6/files/templates/suggestionindex.json new file mode 100644 index 000000000..cb32559b1 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/suggestionindex.json @@ -0,0 +1,784 @@ +{ + "index_patterns": "lp_suggestion_index_v1", + "settings": { + "index": { + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "nGram", + "max_gram": "20" + } + }, + "analyzer": { + "keylower": { + "filter": "lowercase", + "tokenizer": "keyword" + }, + "sg_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "sg_search_analyzer": { + "filter": [ + "standard", + "lowercase" + ], + "type": "custom", + "tokenizer": "standard" + } + } + } + }, + "number_of_shards": 5 + }, + "aliases": { + "lp_suggestion_index_alias": {} + }, + "mappings": { + "sg": { + "dynamic_templates": [ + { + "longs": { + "mapping": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "match_mapping_type": "long" + } + }, + { + "booleans": { + "mapping": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "match_mapping_type": "boolean" + } + }, + { + "doubles": { + "mapping": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "match_mapping_type": "double" + } + }, + { + "dates": { + "mapping": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "match_mapping_type": "date" + } + }, + { + "strings": { + "mapping": { + "copy_to": "all_fields", + "search_analyzer": "sg_search_analyzer", + "analyzer": "sg_index_analyzer", + "type": "text", + "fields": { + "raw": { + "analyzer": "keylower", + "type": "text", + "fielddata": true + } + } + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "all_fields": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "command": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "comment": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "comments": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "createdOn": { + "type": "date", + "fields": { + "raw": { + "type": "date", + "format": "strict_date_optional_time||epoch_millis" + } + }, + "format": "strict_date_optional_time||epoch_millis" + }, + "es_metadata_id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "lastUpdatedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "objectId": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "params": { + "properties": { + "Language": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "ageGroup": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "ageGrups": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "attributions": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "audience": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "code": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "concepts": { + "properties": { + "description": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "relation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "relationName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + } + } + }, + "description": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "domain": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "domains": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "genre": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "gradeLevel": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "language": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "medium": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "methods": { + "properties": { + "description": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "identifier": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "objectType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "relation": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "relationName": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + } + } + }, + "name": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "subject": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "tags": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "templateType": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + } + } + }, + "request": { + "properties": { + "content": { + "properties": { + "comments": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + } + } + } + } + }, + "status": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "suggestedBy": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + }, + "suggestion_id": { + "type": "text", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + }, + "copy_to": [ + "all_fields" + ], + "analyzer": "sg_index_analyzer", + "search_analyzer": "sg_search_analyzer" + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/files/templates/summary.json b/ansible/roles/log-es6/files/templates/summary.json new file mode 100644 index 000000000..016f450b7 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/summary.json @@ -0,0 +1,802 @@ +{ + "index_patterns": "summary-*", + "settings": { + "number_of_shards": 5 + }, + "mappings": { + "events": { + "dynamic": false, + "properties": { + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "uid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "context":{ + "properties": { + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "model": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "granularity": { + "type": "keyword" + }, + "date_range": { + "properties": { + "from": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "to": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + } + } + }, + "dimensions": { + "properties": { + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "gdata": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "sid": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "group_user": { + "type": "boolean" + }, + "anonymous_user": { + "type": "boolean" + }, + "uid": { + "type": "keyword" + }, + "ss_mid": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + }, + "period": { + "type": "integer" + }, + "content_id": { + "type": "keyword" + }, + "item_id": { + "type": "keyword" + }, + "author_id": { + "type": "keyword" + }, + "loc": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "context_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "object_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + } + } + }, + "edata": { + "properties": { + "eks": { + "properties": { + "total_users_count": { + "type": "long" + }, + "total_devices_count": { + "type": "long" + }, + "total_content_count": { + "type": "long" + }, + "content_ids": { + "type": "keyword" + }, + "device_ids": { + "type": "keyword" + }, + "user_ids": { + "type": "keyword" + }, + "contents": { + "type": "keyword" + }, + "unique_users": { + "type": "keyword" + }, + "avg_ts_session": { + "type": "double" + }, + "total_sessions": { + "type": "long" + }, + "avg_interactions_min": { + "type": "double" + }, + "total_interactions": { + "type": "long" + }, + "avg_pageviews": { + "type": "double" + }, + "total_ts": { + "type": "double" + }, + "time_diff": { + "type": "double" + }, + "time_spent": { + "type": "double" + }, + "interact_events_per_min": { + "type": "double" + }, + "interact_events_count": { + "type": "long" + }, + "total_pageviews_count": { + "type": "long" + }, + "page_views_count": { + "type": "long" + }, + "ce_visits": { + "type": "long" + }, + "start_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "end_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncDate": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "timeSpent": { + "type": "double" + }, + "interactEventsPerMin": { + "type": "double" + }, + "mimeType": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "timeDiff": { + "type": "double" + }, + "mode": { + "type": "keyword" + }, + "telemetryVersion": { + "type": "keyword" + }, + "noOfInteractEvents": { + "type": "long" + }, + "interruptTime": { + "type": "double" + }, + "contentCount": { + "type": "long" + }, + "content": { + "type": "keyword" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "first_visit": { + "type": "boolean" + }, + "noOfAttempts": { + "type": "integer" + }, + "m_side_loads": { + "type": "integer" + }, + "m_downloads": { + "type": "integer" + }, + "m_avg_rating": { + "type": "double" + }, + "m_ratings": { + "type": "nested", + "properties": { + "rating": { + "type": "integer" + }, + "time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + }, + "m_comments": { + "type": "object" + }, + "ce_total_ts": { + "type": "double" + }, + "ce_percent_sessions": { + "type": "double" + }, + "ce_visits_count": { + "type": "long" + }, + "ce_total_visits": { + "type": "long" + }, + "ce_percent_ts": { + "type": "double" + }, + "avg_ts": { + "type": "double" + }, + "inc_res_count": { + "type": "long" + }, + "incorrect_res": { + "type": "nested", + "properties": { + "resp": { + "type": "keyword" + }, + "mmc": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "total_count": { + "type": "long" + }, + "correct_res_count": { + "type": "long" + }, + "correct_res": { + "type": "keyword" + }, + "itemId": { + "type": "keyword" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + }, + "new_user_count": { + "type": "long" + }, + "anon_total_sessions": { + "type": "long" + }, + "anon_total_ts": { + "type": "double" + }, + "anon_avg_ts_session": { + "type": "double" + }, + "percent_new_users_count": { + "type": "double" + }, + "ce_total_sessions": { + "type": "long" + }, + "unique_users_count": { + "type": "long" + }, + "stageVisitCount": { + "type": "long" + }, + "interactEventsCount": { + "type": "long" + }, + "interactEvents": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + } + } + }, + "stageId": { + "type": "keyword" + }, + "visitCount": { + "type": "long" + }, + "num_downloads": { + "type": "long" + }, + "num_sideloads": { + "type": "long" + }, + "avg_depth": { + "type": "double" + }, + "publish_pipeline_summary": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "events_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "eventsSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "screenSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "visitCount": { + "type": "long" + } + } + }, + "activitySummary": { + "type": "nested", + "properties": { + "actType": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "page_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "visit_count": { + "type": "long" + } + } + }, + "env_summary": { + "type": "nested", + "properties": { + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "itemResponses": { + "type": "nested", + "properties": { + "itemId": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + } + } + } + } + } + } + }, + "etags": { + "properties": { + "app": { + "type": "keyword" + }, + "partner": { + "type": "keyword" + }, + "dims": { + "type": "keyword" + } + } + }, + "geoip": { + "properties": { + "location": { + "type": "geo_point" + } + }, + "type": "object" + }, + "contentdata": { + "properties": { + "ageGroup": { + "type": "keyword" + }, + "author": { + "type": "keyword" + }, + "audience": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "collaborators": { + "type": "keyword" + }, + "collections": { + "type": "keyword" + }, + "concepts": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "curriculum": { + "type": "keyword" + }, + "developer": { + "type": "keyword" + }, + "domain": { + "type": "keyword" + }, + "downloadUrl": { + "type": "keyword" + }, + "downloads": { + "type": "long" + }, + "edition": { + "type": "keyword" + }, + "genre": { + "type": "keyword" + }, + "gradeLevel": { + "type": "keyword" + }, + "keywords": { + "type": "keyword" + }, + "me_totalDevices": { + "type": "long" + }, + "me_totalDownloads": { + "type": "long" + }, + "me_totalInteractions": { + "type": "long" + }, + "me_totalRatings": { + "type": "long" + }, + "me_totalSessionsCount": { + "type": "long" + }, + "me_totalSideloads": { + "type": "long" + }, + "me_totalTimespent": { + "type": "long" + }, + "me_totalUsage": { + "type": "long" + }, + "medium": { + "type": "keyword" + }, + "methods": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + }, + "popularity": { + "type": "long" + }, + "portalOwner": { + "type": "keyword" + }, + "publication": { + "type": "keyword" + }, + "publisher": { + "type": "keyword" + }, + "rating": { + "type": "long" + }, + "size": { + "type": "long" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "subject": { + "type": "keyword" + }, + "templateType": { + "type": "keyword" + }, + "theme": { + "type": "keyword" + }, + "words": { + "type": "keyword" + } + } + }, + "itemdata": { + "properties": { + "concepts": { + "type": "keyword" + }, + "createdBy": { + "type": "keyword" + }, + "createdOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "lastUpdatedBy": { + "type": "keyword" + }, + "lastUpdatedOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "media": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "num_answers": { + "type": "long" + }, + "owner": { + "type": "keyword" + }, + "qlevel": { + "type": "keyword" + }, + "question": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "template": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "ldata": { + "properties": { + "country": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "district": { + "type": "keyword" + }, + "city": { + "type": "keyword" + } + } + }, + "@version": { + "type": "keyword" + } + } + } + } +} diff --git a/ansible/roles/log-es6/files/templates/summary_cumulative.json b/ansible/roles/log-es6/files/templates/summary_cumulative.json new file mode 100644 index 000000000..99e01b8bd --- /dev/null +++ b/ansible/roles/log-es6/files/templates/summary_cumulative.json @@ -0,0 +1,802 @@ +{ + "index_patterns": "summary-cumulative-*", + "settings": { + "number_of_shards": 5 + }, + "mappings": { + "events": { + "dynamic": false, + "properties": { + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "uid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "context":{ + "properties": { + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "model": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "granularity": { + "type": "keyword" + }, + "date_range": { + "properties": { + "from": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "to": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + } + } + }, + "dimensions": { + "properties": { + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "gdata": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "sid": { + "type": "keyword" + }, + "channel": { + "type": "keyword" + }, + "group_user": { + "type": "boolean" + }, + "anonymous_user": { + "type": "boolean" + }, + "uid": { + "type": "keyword" + }, + "ss_mid": { + "type": "keyword" + }, + "tag": { + "type": "keyword" + }, + "period": { + "type": "integer" + }, + "content_id": { + "type": "keyword" + }, + "item_id": { + "type": "keyword" + }, + "author_id": { + "type": "keyword" + }, + "loc": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "context_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "object_rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + } + } + }, + "edata": { + "properties": { + "eks": { + "properties": { + "total_users_count": { + "type": "long" + }, + "total_devices_count": { + "type": "long" + }, + "total_content_count": { + "type": "long" + }, + "content_ids": { + "type": "keyword" + }, + "device_ids": { + "type": "keyword" + }, + "user_ids": { + "type": "keyword" + }, + "contents": { + "type": "keyword" + }, + "unique_users": { + "type": "keyword" + }, + "avg_ts_session": { + "type": "double" + }, + "total_sessions": { + "type": "long" + }, + "avg_interactions_min": { + "type": "double" + }, + "total_interactions": { + "type": "long" + }, + "avg_pageviews": { + "type": "double" + }, + "total_ts": { + "type": "double" + }, + "time_diff": { + "type": "double" + }, + "time_spent": { + "type": "double" + }, + "interact_events_per_min": { + "type": "double" + }, + "interact_events_count": { + "type": "long" + }, + "total_pageviews_count": { + "type": "long" + }, + "page_views_count": { + "type": "long" + }, + "ce_visits": { + "type": "long" + }, + "start_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "end_time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "syncDate": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "timeSpent": { + "type": "double" + }, + "interactEventsPerMin": { + "type": "double" + }, + "mimeType": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "timeDiff": { + "type": "double" + }, + "mode": { + "type": "keyword" + }, + "telemetryVersion": { + "type": "keyword" + }, + "noOfInteractEvents": { + "type": "long" + }, + "interruptTime": { + "type": "double" + }, + "contentCount": { + "type": "long" + }, + "content": { + "type": "keyword" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "first_visit": { + "type": "boolean" + }, + "noOfAttempts": { + "type": "integer" + }, + "m_side_loads": { + "type": "integer" + }, + "m_downloads": { + "type": "integer" + }, + "m_avg_rating": { + "type": "double" + }, + "m_ratings": { + "type": "nested", + "properties": { + "rating": { + "type": "integer" + }, + "time": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + } + } + }, + "m_comments": { + "type": "object" + }, + "ce_total_ts": { + "type": "double" + }, + "ce_percent_sessions": { + "type": "double" + }, + "ce_visits_count": { + "type": "long" + }, + "ce_total_visits": { + "type": "long" + }, + "ce_percent_ts": { + "type": "double" + }, + "avg_ts": { + "type": "double" + }, + "inc_res_count": { + "type": "long" + }, + "incorrect_res": { + "type": "nested", + "properties": { + "resp": { + "type": "keyword" + }, + "mmc": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "total_count": { + "type": "long" + }, + "correct_res_count": { + "type": "long" + }, + "correct_res": { + "type": "keyword" + }, + "itemId": { + "type": "keyword" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + }, + "new_user_count": { + "type": "long" + }, + "anon_total_sessions": { + "type": "long" + }, + "anon_total_ts": { + "type": "double" + }, + "anon_avg_ts_session": { + "type": "double" + }, + "percent_new_users_count": { + "type": "double" + }, + "ce_total_sessions": { + "type": "long" + }, + "unique_users_count": { + "type": "long" + }, + "stageVisitCount": { + "type": "long" + }, + "interactEventsCount": { + "type": "long" + }, + "interactEvents": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + } + } + }, + "stageId": { + "type": "keyword" + }, + "visitCount": { + "type": "long" + }, + "num_downloads": { + "type": "long" + }, + "num_sideloads": { + "type": "long" + }, + "avg_depth": { + "type": "double" + }, + "publish_pipeline_summary": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "events_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "eventsSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "count": { + "type": "long" + } + } + }, + "screenSummary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "visitCount": { + "type": "long" + } + } + }, + "activitySummary": { + "type": "nested", + "properties": { + "actType": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "page_summary": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "visit_count": { + "type": "long" + } + } + }, + "env_summary": { + "type": "nested", + "properties": { + "env": { + "type": "keyword" + }, + "time_spent": { + "type": "double" + }, + "count": { + "type": "long" + } + } + }, + "itemResponses": { + "type": "nested", + "properties": { + "itemId": { + "type": "keyword" + }, + "timeSpent": { + "type": "double" + }, + "exTimeSpent": { + "type": "double" + }, + "score": { + "type": "integer" + }, + "time_stamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "pass": { + "type": "keyword" + }, + "qtitle": { + "type": "keyword" + }, + "qdesc": { + "type": "keyword" + } + } + } + } + } + } + }, + "etags": { + "properties": { + "app": { + "type": "keyword" + }, + "partner": { + "type": "keyword" + }, + "dims": { + "type": "keyword" + } + } + }, + "geoip": { + "properties": { + "location": { + "type": "geo_point" + } + }, + "type": "object" + }, + "contentdata": { + "properties": { + "ageGroup": { + "type": "keyword" + }, + "author": { + "type": "keyword" + }, + "audience": { + "type": "keyword" + }, + "code": { + "type": "keyword" + }, + "collaborators": { + "type": "keyword" + }, + "collections": { + "type": "keyword" + }, + "concepts": { + "type": "keyword" + }, + "contentType": { + "type": "keyword" + }, + "curriculum": { + "type": "keyword" + }, + "developer": { + "type": "keyword" + }, + "domain": { + "type": "keyword" + }, + "downloadUrl": { + "type": "keyword" + }, + "downloads": { + "type": "long" + }, + "edition": { + "type": "keyword" + }, + "genre": { + "type": "keyword" + }, + "gradeLevel": { + "type": "keyword" + }, + "keywords": { + "type": "keyword" + }, + "me_totalDevices": { + "type": "long" + }, + "me_totalDownloads": { + "type": "long" + }, + "me_totalInteractions": { + "type": "long" + }, + "me_totalRatings": { + "type": "long" + }, + "me_totalSessionsCount": { + "type": "long" + }, + "me_totalSideloads": { + "type": "long" + }, + "me_totalTimespent": { + "type": "long" + }, + "me_totalUsage": { + "type": "long" + }, + "medium": { + "type": "keyword" + }, + "methods": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "owner": { + "type": "keyword" + }, + "popularity": { + "type": "long" + }, + "portalOwner": { + "type": "keyword" + }, + "publication": { + "type": "keyword" + }, + "publisher": { + "type": "keyword" + }, + "rating": { + "type": "long" + }, + "size": { + "type": "long" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "subject": { + "type": "keyword" + }, + "templateType": { + "type": "keyword" + }, + "theme": { + "type": "keyword" + }, + "words": { + "type": "keyword" + } + } + }, + "itemdata": { + "properties": { + "concepts": { + "type": "keyword" + }, + "createdBy": { + "type": "keyword" + }, + "createdOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "keywords": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "lastUpdatedBy": { + "type": "keyword" + }, + "lastUpdatedOn": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "media": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "num_answers": { + "type": "long" + }, + "owner": { + "type": "keyword" + }, + "qlevel": { + "type": "keyword" + }, + "question": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "template": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "ldata": { + "properties": { + "country": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "district": { + "type": "keyword" + }, + "city": { + "type": "keyword" + } + } + }, + "@version": { + "type": "keyword" + } + } + } + } +} diff --git a/ansible/roles/log-es6/files/templates/telemetry.json b/ansible/roles/log-es6/files/templates/telemetry.json new file mode 100644 index 000000000..afe952232 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/telemetry.json @@ -0,0 +1,430 @@ +{ + "index_patterns" : "telemetry-*", + "settings" : { + "number_of_shards" : 5 + }, + "mappings" : { + "events" : { + "dynamic": false, + "properties": { + "@timestamp": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "@version": { + "type": "keyword" + }, + "eid": { + "type": "keyword" + }, + "ets": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "mid": { + "type": "keyword" + }, + "ts": { + "format": "strict_date_optional_time||epoch_millis", + "type": "date" + }, + "ver": { + "type": "keyword" + }, + "actor": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "edata": { + "properties": { + "comments": { + "type": "keyword" + }, + "correlationid": { + "type": "keyword" + }, + "duration": { + "type": "double" + }, + "data": { + "type": "keyword" + }, + "dir": { + "type": "keyword" + }, + "errtype": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "level": { + "type": "keyword" + }, + "loc": { + "type": "keyword" + }, + "message": { + "type": "keyword" + }, + "mode": { + "type": "keyword" + }, + "pass": { + "type": "keyword" + }, + "prevstate": { + "type": "keyword" + }, + "pageid": { + "type": "keyword" + }, + "query": { + "type": "keyword" + }, + "rating": { + "type": "double" + }, + "score": { + "type": "double" + }, + "size": { + "type": "double" + }, + "state": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + }, + "items": { + "type": "nested", + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "origin": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + } + } + }, + "item": { + "properties": { + "id": { + "type": "keyword" + }, + "maxscore": { + "type": "long" + }, + "exlength": { + "type": "long" + }, + "uri": { + "type": "keyword" + }, + "desc": { + "type": "keyword" + }, + "title": { + "type": "keyword" + } + } + }, + "target": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "category": { + "type": "keyword" + }, + "parent": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + } + } + }, + "visits": { + "type": "nested", + "properties": { + "objid": { + "type": "keyword" + }, + "objtype": { + "type": "keyword" + } + } + }, + "plugin": { + "properties": { + "id": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "category": { + "type": "keyword" + } + } + }, + "object": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "uaspec": { + "properties": { + "agent": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "system": { + "type": "keyword" + }, + "platform": { + "type": "keyword" + }, + "raw": { + "type": "keyword" + } + } + }, + "dspec": { + "properties": { + "camera": { + "type": "keyword" + }, + "cpu": { + "type": "keyword" + }, + "edisk": { + "type": "double" + }, + "id": { + "type": "keyword" + }, + "idisk": { + "type": "double" + }, + "make": { + "type": "keyword" + }, + "mem": { + "type": "double" + }, + "os": { + "type": "keyword" + }, + "scrn": { + "type": "double" + }, + "sims": { + "type": "double" + } + } + } + } + }, + "context": { + "properties": { + "channel": { + "type": "keyword" + }, + "env": { + "type": "keyword" + }, + "sid": { + "type": "keyword" + }, + "did": { + "type": "keyword" + }, + "pdata": { + "properties": { + "id": { + "type": "keyword" + }, + "pid": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + } + } + }, + "rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + }, + "cdata": { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "id": { + "type": "keyword" + } + } + } + } + }, + "object": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "parentid": { + "type": "keyword" + }, + "parenttype": { + "type": "keyword" + }, + "ver": { + "type": "keyword" + }, + "rollup": { + "properties": { + "l1": { + "type": "keyword" + }, + "l2": { + "type": "keyword" + }, + "l3": { + "type": "keyword" + }, + "l4": { + "type": "keyword" + } + } + } + } + }, + "ldata": { + "properties": { + "country": { + "type": "keyword" + }, + "state": { + "type": "keyword" + }, + "district": { + "type": "keyword" + }, + "city": { + "type": "keyword" + } + } + }, + "metadata": { + "properties": { + "source": { + "type": "keyword" + }, + "index_name": { + "type": "keyword" + }, + "index_type": { + "type": "keyword" + }, + "source_eid": { + "type": "keyword" + }, + "source_mid": { + "type": "keyword" + }, + "pump": { + "type": "keyword" + } + } + }, + "flags": { + "properties": { + "v2_converted": { + "type": "boolean" + }, + "dd_processed": { + "type": "boolean" + }, + "tv_processed": { + "type": "boolean" + }, + "ldata_retrieved": { + "type": "boolean" + } + } + } + } + } + } +} diff --git a/ansible/roles/log-es6/files/templates/vocabulary.json b/ansible/roles/log-es6/files/templates/vocabulary.json new file mode 100644 index 000000000..d50cc7ec0 --- /dev/null +++ b/ansible/roles/log-es6/files/templates/vocabulary.json @@ -0,0 +1,130 @@ +{ + "index_patterns": "vocabularyterm", + "settings": { + "index": { + "analysis": { + "filter": { + "mynGram": { + "token_chars": [ + "letter", + "digit", + "whitespace", + "punctuation", + "symbol" + ], + "min_gram": "1", + "type": "edge_ngram", + "max_gram": "20" + } + }, + "analyzer": { + "keylower": { + "filter": "lowercase", + "tokenizer": "keyword" + }, + "vt_index_analyzer": { + "filter": [ + "lowercase", + "mynGram" + ], + "type": "custom", + "tokenizer": "standard" + }, + "vt_search_analyzer": { + "filter": [ + "standard", + "lowercase" + ], + "type": "custom", + "tokenizer": "standard" + } + } + } + }, + "number_of_shards": 5 + }, + "mappings": { + "vt": { + "dynamic_templates": [ + { + "longs": { + "mapping": { + "type": "long", + "fields": { + "raw": { + "type": "long" + } + } + }, + "match_mapping_type": "long" + } + }, + { + "booleans": { + "mapping": { + "type": "boolean", + "fields": { + "raw": { + "type": "boolean" + } + } + }, + "match_mapping_type": "boolean" + } + }, + { + "doubles": { + "mapping": { + "type": "double", + "fields": { + "raw": { + "type": "double" + } + } + }, + "match_mapping_type": "double" + } + }, + { + "dates": { + "mapping": { + "type": "date", + "fields": { + "raw": { + "type": "date" + } + } + }, + "match_mapping_type": "date" + } + }, + { + "strings": { + "mapping": { + "type": "text", + "copy_to": "all_fields", + "analyzer": "vt_index_analyzer", + "search_analyzer": "vt_search_analyzer", + "fields": { + "raw": { + "type": "text", + "fielddata": true, + "analyzer": "keylower" + } + } + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "all_fields": { + "type": "text", + "fielddata": true, + "analyzer": "vt_index_analyzer", + "search_analyzer": "vt_search_analyzer" + } + } + } + } +} \ No newline at end of file diff --git a/ansible/roles/log-es6/filter_plugins/custom.py b/ansible/roles/log-es6/filter_plugins/custom.py new file mode 100644 index 000000000..22177cdd1 --- /dev/null +++ b/ansible/roles/log-es6/filter_plugins/custom.py @@ -0,0 +1,57 @@ +__author__ = 'dale mcdiarmid' + +import re +import os.path +from six import string_types + +def modify_list(values=[], pattern='', replacement='', ignorecase=False): + ''' Perform a `re.sub` on every item in the list''' + if ignorecase: + flags = re.I + else: + flags = 0 + _re = re.compile(pattern, flags=flags) + return [_re.sub(replacement, value) for value in values] + +def append_to_list(values=[], suffix=''): + if isinstance(values, string_types): + values = values.split(',') + return [str(value+suffix) for value in values] + +def array_to_str(values=[],separator=','): + return separator.join(values) + +def extract_role_users(users={},exclude_users=[]): + role_users=[] + for user,details in users.iteritems(): + if user not in exclude_users and "roles" in details: + for role in details["roles"]: + role_users.append(role+":"+user) + return role_users + +def filename(filename=''): + return os.path.splitext(os.path.basename(filename))[0] + +def remove_reserved(user_roles={}): + not_reserved = [] + for user_role,details in user_roles.items(): + if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]: + not_reserved.append(user_role) + return not_reserved + +def filter_reserved(users_role={}): + reserved = [] + for user_role,details in users_role.items(): + if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]: + reserved.append(user_role) + return reserved + +class FilterModule(object): + def filters(self): + return {'modify_list': modify_list, + 'append_to_list':append_to_list, + 'filter_reserved':filter_reserved, + 'array_to_str':array_to_str, + 'extract_role_users':extract_role_users, + 'remove_reserved':remove_reserved, + 'filename':filename} \ No newline at end of file diff --git a/ansible/roles/log-es6/handlers/main.yml b/ansible/roles/log-es6/handlers/main.yml new file mode 100644 index 000000000..d71397a39 --- /dev/null +++ b/ansible/roles/log-es6/handlers/main.yml @@ -0,0 +1,14 @@ + +- name: reload systemd configuration + become: yes + command: systemctl daemon-reload + +# Restart service and ensure it is enabled + +- name: restart elasticsearch + become: yes + service: name={{instance_init_script | basename}} state=restarted enabled=yes + when: + - es_restart_on_change + - es_start_service + register: es_restarted diff --git a/ansible/roles/log-es6/meta/main.yml b/ansible/roles/log-es6/meta/main.yml new file mode 100644 index 000000000..aeecec776 --- /dev/null +++ b/ansible/roles/log-es6/meta/main.yml @@ -0,0 +1,25 @@ +--- + +allow_duplicates: yes + +galaxy_info: + author: Robin Clarke, Jakob Reiter, Dale McDiarmid + description: Elasticsearch for Linux + company: "Elastic.co" + license: "license (Apache)" + min_ansible_version: 2.3.2 + platforms: + - name: EL + versions: + - 6 + - 7 + - name: Debian + versions: + - all + - name: Ubuntu + versions: + - all + categories: + - system + +dependencies: [] diff --git a/ansible/roles/log-es6/tasks/azure-plugin.yml b/ansible/roles/log-es6/tasks/azure-plugin.yml new file mode 100644 index 000000000..53a55df30 --- /dev/null +++ b/ansible/roles/log-es6/tasks/azure-plugin.yml @@ -0,0 +1,11 @@ +- name: Check if plugin already installed or not + shell: "curl -s localhost:9200/_cat/plugins | awk '{print $2}' |head -1" + register: plugin_output + +- name: check the output + debug: + var: plugin_output + +- name: Install plugin + shell: ES_PATH_CONF=/etc/elasticsearch/"{{ es_instance_name }}" /usr/share/elasticsearch/bin/elasticsearch-plugin install -b repository-azure + when: plugin_output.stdout != "repository-azure" diff --git a/ansible/roles/log-es6/tasks/elasticsearch-Debian-version-lock.yml b/ansible/roles/log-es6/tasks/elasticsearch-Debian-version-lock.yml new file mode 100644 index 000000000..d9fdd698b --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-Debian-version-lock.yml @@ -0,0 +1,6 @@ +--- +- name: Debian - hold elasticsearch version + become: yes + command: apt-mark hold elasticsearch + register: hold_elasticsearch_result + changed_when: "hold_elasticsearch_result.stdout != 'elasticsearch was already set on hold.'" diff --git a/ansible/roles/log-es6/tasks/elasticsearch-Debian.yml b/ansible/roles/log-es6/tasks/elasticsearch-Debian.yml new file mode 100644 index 000000000..9530fc740 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-Debian.yml @@ -0,0 +1,66 @@ +--- + +- name: set fact force_install to no + set_fact: force_install=no + +- name: set fact force_install to yes + set_fact: force_install=yes + when: es_allow_downgrades + +- name: Debian - Install apt-transport-https to support https APT downloads + become: yes + apt: name=apt-transport-https state=present + when: es_use_repository + +- name: Debian - Add Elasticsearch repository key + become: yes + apt_key: url="{{ es_apt_key }}" state=present + when: es_use_repository and es_apt_key + +- name: Debian - Add elasticsearch repository + become: yes + apt_repository: repo={{ item.repo }} state={{ item.state}} + with_items: + - { repo: "{{ es_apt_url_old }}", state: "absent" } + - { repo: "{{ es_apt_url }}", state: "present" } + when: es_use_repository + +- name: Debian - Include versionlock + include: elasticsearch-Debian-version-lock.yml + when: es_version_lock + + #Copy the elasticsearch mandatory variable + +- name: Create es group + group: name={{ es_group }} state=present + +- name: Create es user + user: name={{ es_user }} group={{ es_group }} state=present + +- name: Copy the elasticsearch mandatory variables + become: yes + template: + src: environment.j2 + dest: "{{ env_dir }}/elasticsearch" + owner: "{{ es_user }}" + group: "{{ es_group }}" + mode: 0660 + force: yes + +- name: Debian - Ensure elasticsearch is installed + become: yes + apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400 + when: es_use_repository + register: debian_elasticsearch_install_from_repo + notify: restart elasticsearch + +- name: Debian - Download elasticsearch from url + get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no + when: not es_use_repository + +- name: Debian - Ensure elasticsearch is installed from downloaded package + become: yes + apt: deb=/tmp/elasticsearch-{{ es_version }}.deb + when: not es_use_repository + register: elasticsearch_install_from_package + notify: restart elasticsearch diff --git a/ansible/roles/log-es6/tasks/elasticsearch-RedHat-version-lock.yml b/ansible/roles/log-es6/tasks/elasticsearch-RedHat-version-lock.yml new file mode 100644 index 000000000..b5711a2f3 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-RedHat-version-lock.yml @@ -0,0 +1,7 @@ +--- +- name: RedHat - install yum-version-lock + become: yes + yum: name=yum-plugin-versionlock state=present update_cache=yes +- name: RedHat - lock elasticsearch version + become: yes + shell: yum versionlock delete 0:elasticsearch* ; yum versionlock add elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} diff --git a/ansible/roles/log-es6/tasks/elasticsearch-RedHat.yml b/ansible/roles/log-es6/tasks/elasticsearch-RedHat.yml new file mode 100644 index 000000000..588f0cec0 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-RedHat.yml @@ -0,0 +1,31 @@ +--- +- name: Ensure libselinux-python on CentOS 6.x + become: yes + yum: name=libselinux-python state=present update_cache=yes + when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" ) + +- name: RedHat - add Elasticsearch repo + become: yes + template: src=elasticsearch.repo dest=/etc/yum.repos.d/elasticsearch-{{ es_major_version }}.repo + when: es_use_repository + +- name: RedHat - include versionlock + include: elasticsearch-RedHat-version-lock.yml + when: es_version_lock + +- name: RedHat - Install Elasticsearch + become: yes + yum: name=elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %} state=present update_cache=yes + when: es_use_repository + register: redhat_elasticsearch_install_from_repo + notify: restart elasticsearch + until: '"failed" not in redhat_elasticsearch_install_from_repo' + retries: 5 + delay: 10 + +- name: RedHat - Install Elasticsearch from url + become: yes + yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present + when: not es_use_repository + register: elasticsearch_install_from_package + notify: restart elasticsearch diff --git a/ansible/roles/log-es6/tasks/elasticsearch-config.yml b/ansible/roles/log-es6/tasks/elasticsearch-config.yml new file mode 100644 index 000000000..ff788d1c5 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-config.yml @@ -0,0 +1,111 @@ +--- +# Configure Elasticsearch Node + +#Create required directories +- name: Create Directories + become: yes + file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} + with_items: + - "{{pid_dir}}" + - "{{log_dir}}" + - "{{conf_dir}}" + +- name: Create Data Directories + become: yes + file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }} + with_items: + - "{{data_dirs}}" + +#Copy the config template +- name: Copy Configuration File + become: yes + template: src=elasticsearch.yml.j2 dest={{conf_dir}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + register: system_change + notify: restart elasticsearch + +#Copy the instance specific default file +- name: Copy Default File for Instance + become: yes + template: src=elasticsearch.j2 dest={{instance_default_file}} mode=0644 force=yes + notify: restart elasticsearch + +#Copy the instance specific init file +- name: Copy Debian Init File for Instance + become: yes + template: src=init/debian/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes + when: ansible_os_family == 'Debian' and not use_system_d + notify: restart elasticsearch + +#Copy the instance specific init file +- name: Copy Redhat Init File for Instance + become: yes + template: src=init/redhat/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes + when: ansible_os_family == 'RedHat' and not use_system_d + notify: restart elasticsearch + +#Copy the systemd specific file if systemd is installed +- name: Copy Systemd File for Instance + become: yes + template: src=systemd/elasticsearch.j2 dest={{instance_sysd_script}} mode=0644 force=yes + when: use_system_d + notify: + - reload systemd configuration + - restart elasticsearch + +#Copy the logging.yml +- name: Copy log4j2.properties File for Instance + become: yes + template: src={{es_config_log4j2}} dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + +- name: Get information on the RAM + set_fact: + log_es_heap_size: "{% if ((ansible_memory_mb.real.total/(2*1024))|int) > 26 %}26g{% else %}{{(((ansible_memory_mb.real.total/(2*1024))+1)|int)}}g{% endif %}" + when: log_es_heap_size is not defined + +- name: Copy jvm.options File for Instance + become: yes + template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + notify: restart elasticsearch + +#Clean up un-wanted package scripts to avoid confusion + +- name: Delete Default Init + become: yes + file: dest=/etc/init.d/elasticsearch state=absent + +- name: Create empty default environment file + become: yes + changed_when: False + copy: + dest: /etc/default/elasticsearch + content: '' + when: ansible_os_family == 'Debian' + +- name: Create empty default environment file + become: yes + changed_when: False + copy: + dest: /etc/sysconfig/elasticsearch + content: '' + when: ansible_os_family == 'RedHat' + +- name: Delete Default Sysconfig File + become: yes + file: dest="{{ sysd_script }}" state=absent + +- name: Delete Default Configuration File + become: yes + file: dest=/etc/elasticsearch/elasticsearch.yml state=absent + +- name: Delete Default Logging File + become: yes + file: dest=/etc/elasticsearch/logging.yml state=absent + +- name: Delete Default Logging File + become: yes + file: dest=/etc/elasticsearch/log4j2.properties state=absent + +- name: Delete Default JVM Options File + become: yes + file: dest=/etc/elasticsearch/jvm.options state=absent diff --git a/ansible/roles/log-es6/tasks/elasticsearch-optional-user.yml b/ansible/roles/log-es6/tasks/elasticsearch-optional-user.yml new file mode 100644 index 000000000..d8f787e4b --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-optional-user.yml @@ -0,0 +1,24 @@ +--- +#Add the elasticsearch user before installing from packages. +- name: Ensure optional elasticsearch group is created with the correct id. + become: yes + #Restart if these change + notify: restart elasticsearch + group: + state: present + name: "{{ es_group }}" + system: yes + gid: "{{ es_group_id }}" + +- name: Ensure optional elasticsearch user is created with the correct id. + become: yes + #Restart if these change + notify: restart elasticsearch + user: + state: present + name: "{{ es_user }}" + comment: elasticsearch system user + system: yes + createhome: no + uid: "{{ es_user_id }}" + group: "{{ es_group }}" diff --git a/ansible/roles/log-es6/tasks/elasticsearch-parameters.yml b/ansible/roles/log-es6/tasks/elasticsearch-parameters.yml new file mode 100644 index 000000000..779aabb6d --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-parameters.yml @@ -0,0 +1,75 @@ +# Check for mandatory parameters + +- name: fail when es_instance is not defined + fail: msg="es_instance_name must be specified and cannot be blank" + when: es_instance_name is not defined or es_instance_name == '' + +- name: fail when es_proxy_port is not defined or is blank + fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined" + when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '') + +- name: debug message + debug: msg="WARNING - It is recommended you specify the parameter 'http.port'" + when: es_config['http.port'] is not defined + +- name: debug message + debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'" + when: es_config['transport.tcp.port'] is not defined + +- name: debug message + debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'" + when: es_config['discovery.zen.ping.unicast.hosts'] is not defined + +#If the user attempts to lock memory they must specify a heap size +- name: fail when heap size is not specified when using memory lock + fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified" + when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and log_es_heap_size is not defined + +#Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work +- name: fail when api credentials are not declared when using security + fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations" + when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined + +- name: set fact file_reserved_users + set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }} + when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0) + +- name: fail when changing users through file realm + fail: + msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM." + when: file_reserved_users | default([]) | length > 0 + +- name: set fact instance_default_file + set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}} +- name: set fact instance_init_script + set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}} +- name: set fact conf_dir + set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}} +- name: set fact m_lock_enabled + set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }} + +#TODO - if transport.host is not local maybe error on boostrap checks + + +#Use systemd for the following distributions: +#Ubuntu 15 and up +#Debian 8 and up +#Centos 7 and up +#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied. + +- name: set fact use_system_d + set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }} + +- name: set fact instance_sysd_script + set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}} + when: use_system_d +#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN. + +- name: set fact instance_suffix + set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }} +- name: set fact pid_dir + set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}} +- name: set fact log_dir + set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}} +- name: set fact log_dir + set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }} diff --git a/ansible/roles/log-es6/tasks/elasticsearch-plugins.yml b/ansible/roles/log-es6/tasks/elasticsearch-plugins.yml new file mode 100644 index 000000000..5d4e2d815 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-plugins.yml @@ -0,0 +1,82 @@ +--- + +# es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed +# i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting. +- name: set fact es_plugins_reinstall to true + set_fact: es_plugins_reinstall=true + when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none + +- name: set fact list_command + set_fact: list_command="" +#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent. +- name: set fact list_command check for x-pack + set_fact: list_command="| grep -vE 'x-pack'" + when: not es_plugins_reinstall + +#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version. +- name: Check installed elasticsearch plugins + become: yes + shell: "ls {{es_home}}/plugins {{list_command}}" + register: installed_plugins + changed_when: False + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +#if es_plugins_reinstall is set to true we remove ALL plugins +- name: set fact plugins_to_remove to install_plugins.stdout_lines + set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}" + when: es_plugins_reinstall + +#if the plugins listed are different than those requested, we remove those installed but not listed in the config +- name: set fact plugins_to_remove to delete plugins installed but not listed in es_plugins + set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}" + when: not es_plugins_reinstall + +#if es_plugins_reinstall is set to true we (re)install ALL plugins +- name: set fact plugins_to_install to es_plugins + set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}" + when: es_plugins_reinstall + +#if the plugins listed are different than those requested, we install those not installed but listed in the config +- name: set fact to plugins_to_install to those in es_config but not installed + set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}" + when: not es_plugins_reinstall + +# This removes any currently installed plugins (to prevent errors when reinstalling) +- name: Remove elasticsearch plugins + become: yes + command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent" + ignore_errors: yes + with_items: "{{ plugins_to_remove | default([]) }}" + notify: restart elasticsearch + register: plugin_removed + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +- name: Install elasticsearch plugins + become: yes + command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent" + register: plugin_installed + failed_when: "'ERROR' in plugin_installed.stdout" + changed_when: plugin_installed.rc == 0 + with_items: "{{ es_plugins }}" + when: item.plugin in plugins_to_install + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }} {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}" + until: plugin_installed.rc == 0 + retries: 5 + delay: 5 + +#Set permissions on plugins directory +- name: Set Plugin Directory Permissions + become: yes + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes diff --git a/ansible/roles/log-es6/tasks/elasticsearch-scripts.yml b/ansible/roles/log-es6/tasks/elasticsearch-scripts.yml new file mode 100644 index 000000000..e38c3b4c4 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-scripts.yml @@ -0,0 +1,26 @@ +--- + +- name: set fact es_script_dir + set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}} + tags: + - always + +- name: set fact es_script_dir when path.scripts + set_fact: es_script_dir={{es_config['path.scripts']}} + when: es_config['path.scripts'] is defined + tags: + - always + +- name: Create script dir + become: yes + file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes + +- name: Copy default scripts to elasticsearch + become: yes + copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} + when: es_scripts_fileglob is not defined + +- name: Copy scripts to elasticsearch + become: yes + copy: src={{ item }} dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} + with_fileglob: "{{ es_scripts_fileglob | default('') }}" diff --git a/ansible/roles/log-es6/tasks/elasticsearch-template.yml b/ansible/roles/log-es6/tasks/elasticsearch-template.yml new file mode 100644 index 000000000..6059c7d72 --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch-template.yml @@ -0,0 +1,40 @@ +--- +- name: ensure templates dir is created + file: + path: /etc/elasticsearch/templates + state: directory + owner: "{{ es_user }}" + group: "{{ es_group }}" + +- name: Copy templates to elasticsearch + copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }} + register: load_templates + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + +- name: Install templates without auth + uri: + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" + method: PUT + status_code: 200 + body_format: json + body: "{{ lookup('file', item) }}" + when: load_templates.changed and es_start_service and not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + run_once: True + +- name: Install templates with auth + uri: + url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}" + method: PUT + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + body_format: json + body: "{{ lookup('file', item) }}" + when: load_templates.changed and es_start_service and es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features + with_fileglob: + - "{{ es_templates_fileglob | default('') }}" + run_once: True diff --git a/ansible/roles/log-es6/tasks/elasticsearch.yml b/ansible/roles/log-es6/tasks/elasticsearch.yml new file mode 100644 index 000000000..e2361d49b --- /dev/null +++ b/ansible/roles/log-es6/tasks/elasticsearch.yml @@ -0,0 +1,13 @@ +--- + +- name: Include optional user and group creation. + when: (es_user_id is defined) and (es_group_id is defined) + include: elasticsearch-optional-user.yml + +- name: Include specific Elasticsearch + include: elasticsearch-Debian.yml + when: ansible_os_family == 'Debian' + +- name: Include specific Elasticsearch + include: elasticsearch-RedHat.yml + when: ansible_os_family == 'RedHat' diff --git a/ansible/roles/log-es6/tasks/java.yml b/ansible/roles/log-es6/tasks/java.yml new file mode 100644 index 000000000..a4ecac04a --- /dev/null +++ b/ansible/roles/log-es6/tasks/java.yml @@ -0,0 +1,51 @@ +--- + +- name: set fact java_state to present + set_fact: java_state="present" + +- name: set fact java_state to latest + set_fact: java_state="latest" + when: update_java == true + +- name: RedHat - Ensure Java is installed + become: yes + yum: name={{ java }} state={{java_state}} + when: ansible_os_family == 'RedHat' + +- name: Get the installed java path + shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0" + register: java_full_path + failed_when: False + changed_when: False + when: ansible_os_family == 'RedHat' + +- name: correct java version selected + alternatives: + name: java + path: "{{ java_full_path.stdout }}" + link: /usr/bin/java + when: ansible_os_family == 'RedHat' and java_full_path is defined + +- name: Refresh java repo + become: yes + apt: update_cache=yes + changed_when: false + when: ansible_os_family == 'Debian' + +- name: Debian - Ensure Java is installed + become: yes + apt: name={{ java }} state={{java_state}} + when: ansible_os_family == 'Debian' + +- name: register open_jdk version + shell: java -version 2>&1 | grep OpenJDK + register: open_jdk + ignore_errors: yes + changed_when: false + +#https://github.com/docker-library/openjdk/issues/19 - ensures tests pass due to java 8 broken certs +- name: refresh the java ca-certificates + become: yes + command: /var/lib/dpkg/info/ca-certificates-java.postinst configure + when: ansible_distribution == 'Ubuntu' and open_jdk.rc == 0 + changed_when: false diff --git a/ansible/roles/log-es6/tasks/main.yml b/ansible/roles/log-es6/tasks/main.yml new file mode 100644 index 000000000..7bf1a214a --- /dev/null +++ b/ansible/roles/log-es6/tasks/main.yml @@ -0,0 +1,93 @@ +--- +- name: os-specific vars + include_vars: "{{ansible_os_family}}.yml" + tags: + - always + +- name: check-set-parameters + include: elasticsearch-parameters.yml + tags: + - always + +- name: include elasticsearch.yml + include: elasticsearch.yml + tags: + - install + +- name: include elasticsearch-config.yml + include: elasticsearch-config.yml + tags: + - config + +- name: include elasticsearch-scripts.yml + include: elasticsearch-scripts.yml + when: es_scripts + tags: + - scripts + +- name: include elasticsearch-plugins.yml + include: elasticsearch-plugins.yml + when: es_plugins is defined or es_plugins_reinstall + tags: + - plugins + +# Install Elasticsearch plugin for backups +- name: include elasticsearch plugin install + include: azure-plugin.yml + +#We always execute xpack as we may need to remove features +- name: include xpack/elasticsearch-xpack.yml + include: xpack/elasticsearch-xpack.yml + tags: + - xpack + +- name: flush handlers + meta: flush_handlers + +- name: Make sure elasticsearch is started + service: name={{instance_init_script | basename}} state=started enabled=yes + when: es_start_service + +- name: Wait for elasticsearch to startup + wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1 + when: es_restarted is defined and es_restarted.changed and es_start_service + +- name: set fact manage_native_realm to false + set_fact: manage_native_realm=false + +- name: set fact manage_native_realm to true + set_fact: manage_native_realm=true + when: es_start_service and (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined)) + +# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up +- name: Wait 15 seconds for the Native Relm to come up + pause: seconds=15 + when: manage_native_realm + +- name: activate-license + include: ./xpack/security/elasticsearch-xpack-activation.yml + when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != '' + +#perform security actions here now elasticsearch is started +- name: include xpack/security/elasticsearch-security-native.yml + include: ./xpack/security/elasticsearch-security-native.yml + when: manage_native_realm + +#We also do after the native realm to ensure any changes are applied here first and its denf up. +- name: include elasticsearch-template.yml + include: elasticsearch-template.yml + when: es_templates + tags: + - templates + +# Add ES Plugin credentials for backups +- name: Add ES Backup variables to elasticsearch to have backups enabled + lineinfile: + path: '{{conf_dir}}/elasticsearch.yml' + line: 'cloud.azure.storage.default.{{item.key}}: {{item.value}}' + regexp: "cloud.azure.storage.default.{{ item.key }}.*" + with_items: + - {key: 'account', value: '{{ sunbird_management_storage_account_name }}'} + - {key: 'key', value: '{{ sunbird_management_storage_account_key }}'} + when: sunbird_management_storage_account_name is defined and sunbird_management_storage_account_key is defined + notify: restart elasticsearch diff --git a/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack-install.yml b/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack-install.yml new file mode 100644 index 000000000..522f8161f --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack-install.yml @@ -0,0 +1,68 @@ +--- + +#Test if feature is installed +- name: Test if x-pack is installed + shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack" + become: yes + register: x_pack_installed + changed_when: False + failed_when: "'ERROR' in x_pack_installed.stdout" + check_mode: no + ignore_errors: yes + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Remove X-Pack if installed and its not been requested or the ES version has changed +- name: Remove x-pack plugin + become: yes + command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack" + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + + +#Install plugin if not installed, or the es version has changed (so removed above), and its been requested +- name: Download x-pack from url + get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip + when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined) + +- name: Install x-pack plugin from local + become: yes + command: > + {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip + register: xpack_state + changed_when: xpack_state.rc == 0 + when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + +- name: Delete x-pack zip file + file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent + when: es_xpack_custom_url is defined + +- name: Install x-pack plugin from elastic.co + become: yes + command: > + {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack + register: xpack_state + failed_when: "'ERROR' in xpack_state.stdout" + changed_when: xpack_state.rc == 0 + when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined) + notify: restart elasticsearch + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_INCLUDE: "{{ instance_default_file }}" + ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}" diff --git a/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack.yml b/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack.yml new file mode 100644 index 000000000..ec82a68a6 --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/elasticsearch-xpack.yml @@ -0,0 +1,22 @@ +--- + +- name: set fact es_version_changed + set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }} + +- name: include elasticsearch-xpack-install.yml + include: elasticsearch-xpack-install.yml + +#Security configuration +- name: include security/elasticsearch-security.yml + include: security/elasticsearch-security.yml + +#Add any feature specific configuration here +- name: Set Plugin Directory Permissions + become: yes + file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes + +#Make sure elasticsearch.keystore has correct Permissions +- name: Set elasticsearch.keystore Permissions + become: yes + file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }} + when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>')) diff --git a/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-file.yml b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-file.yml new file mode 100644 index 000000000..8d6f87899 --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-file.yml @@ -0,0 +1,80 @@ +--- +- name: set fact manage_file_users + set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0 + +#List current users +- name: List Users + become: yes + shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}' + register: current_file_users + when: manage_file_users + changed_when: False + +- name: set fact users_to_remove + set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys()) }} + when: manage_file_users + +#Remove users +- name: Remove Users + become: yes + command: > + {{es_home}}/bin/x-pack/users userdel {{item}} + with_items: "{{users_to_remove | default([])}}" + when: manage_file_users + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +- name: set fact users_to_add + set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }} + when: manage_file_users + +#Add users +- name: Add Users + become: yes + command: > + {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}} + with_items: "{{ users_to_add | default([]) }}" + when: manage_file_users + no_log: True + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +#Set passwords for all users declared - Required as the useradd will not change existing user passwords +- name: Set User Passwords + become: yes + command: > + {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}} + with_items: "{{ es_users.file.keys() | default([]) }}" + when: manage_file_users + #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip. + changed_when: False + no_log: True + environment: + CONF_DIR: "{{ conf_dir }}" + ES_PATH_CONF: "{{ conf_dir }}" + ES_HOME: "{{es_home}}" + +- name: set fact users_roles + set_fact: users_roles={{es_users.file | extract_role_users () }} + when: manage_file_users + +#Copy Roles files +- name: Copy roles.yml File for Instance + become: yes + template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_roles is defined and es_roles.file is defined + +#Overwrite users_roles file +- name: Copy User Roles + become: yes + template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes + when: manage_file_users and users_roles | length > 0 + +#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions. +- name: Set Security Directory Permissions Recursive + become: yes + file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes diff --git a/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-native.yml b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-native.yml new file mode 100644 index 000000000..63024fade --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security-native.yml @@ -0,0 +1,191 @@ +--- +- name: set fact change_api_password to false + set_fact: change_api_password=false + +- name: set fact manage_native_users to false + set_fact: manage_native_users=false + +- name: set fact manage_native_users to true + set_fact: manage_native_users=true + when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0 + +- name: set fact manage_native_role to false + set_fact: manage_native_roles=false + +- name: set fact manange_native_roles to true + set_fact: manage_native_roles=true + when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0 + +#If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load + +#List current users +- name: List Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user + method: GET + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: user_list_response + when: manage_native_users + +- name: set fact reserved_users equals user_list_response.json + set_fact: reserved_users={{ user_list_response.json | filter_reserved }} + when: manage_native_users + +#Current users not inc. those reserved +- name: set fact current_users equals user_list_response.json.keys not including reserved + set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }} + when: manage_native_users + +#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param +- name: set fact native_users + set_fact: native_users={{ es_users.native }} + when: manage_native_users + +- name: set fact change_api_password to true + set_fact: change_api_password=true + when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined + +- name: Update API User Password + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password + method: POST + body_format: json + body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: change_api_password + +- name: set fact es_api_basic_auth_password + set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}} + when: change_api_password + +#Identify users that are present in ES but not declared and thus should be removed +- name: set fact users_to_remove + set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }} + when: manage_native_users + +#Delete all non required users NOT inc. reserved +- name: Delete Native Users + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users + with_items: "{{ users_to_remove | default([]) }}" + +- name: set fact users_to_ignore + set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }} + when: manage_native_users + +- name: debug message + debug: + msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}" + when: manage_native_users and users_to_ignore | length > 0 + +#Update password on all reserved users +- name: Update Reserved User Passwords + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password + method: POST + body_format: json + body: "{ \"password\":\"{{native_users[item].password}}\" }" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: native_users[item].password is defined + no_log: True + with_items: "{{ users_to_ignore | default([]) }}" + +- name: set fact users_to_modify + set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }} + when: manage_native_users + +#Overwrite all other users NOT inc. those reserved +- name: Update Non-Reserved Native User Details + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}} + method: POST + body_format: json + body: "{{ native_users[item] | to_json }}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_users + no_log: True + with_items: "{{ users_to_modify | default([]) }}" + +## ROLE CHANGES + +#List current roles not. inc those reserved +- name: List Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role + method: GET + body_format: json + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + status_code: 200 + register: role_list_response + when: manage_native_roles + +- name: set fact reserved roles + set_fact: reserved_roles={{ role_list_response.json | filter_reserved }} + when: manage_native_roles + +- name: set fact current roles + set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }} + when: manage_native_roles + +- name: set fact roles to ignore + set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }} + when: manage_native_roles + +- name: debug message + debug: + msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}" + when: manage_native_roles and roles_to_ignore | length > 0 + +- name: set fact roles_to_remove + set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys() ) }} + when: manage_native_roles + +#Delete all non required roles NOT inc. reserved +- name: Delete Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} + method: DELETE + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles + with_items: "{{roles_to_remove | default([]) }}" + +- name: set fact roles_to_modify + set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }} + when: manage_native_roles + +#Update other roles - NOT inc. reserved roles +- name: Update Native Roles + uri: + url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}} + method: POST + body_format: json + body: "{{ es_roles.native[item] | to_json}}" + status_code: 200 + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + force_basic_auth: yes + when: manage_native_roles + with_items: "{{ roles_to_modify | default([]) }}" diff --git a/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security.yml b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security.yml new file mode 100644 index 000000000..14f4d2038 --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-security.yml @@ -0,0 +1,60 @@ +--- +#Security specific configuration done here + +#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6 + +#Ensure x-pack conf directory is created if necessary +- name: Ensure x-pack conf directory exists (file) + file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }} + changed_when: False + when: + - es_enable_xpack and "security" in es_xpack_features + - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined) + +#-----------------------------Create Bootstrap User----------------------------------- +- name: Check if bootstrap password is set + command: > + {{es_home}}/bin/elasticsearch-keystore list + register: list_keystore + changed_when: False + environment: + ES_PATH_CONF: "{{ conf_dir }}" + when: + - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) + +- name: Create Bootstrap password for elastic user + shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password' + when: + - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines + environment: + ES_PATH_CONF: "{{ conf_dir }}" + no_log: true + +#-----------------------------FILE BASED REALM---------------------------------------- + +- include: elasticsearch-security-file.yml + when: (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined)) + +#-----------------------------ROLE MAPPING ---------------------------------------- + +#Copy Roles files +- name: Copy role_mapping.yml File for Instance + become: yes + template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes + when: es_role_mapping is defined + +#-----------------------------AUTH FILE---------------------------------------- + +- name: Copy message auth key to elasticsearch + become: yes + copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes + when: es_message_auth_file is defined + +#------------------------------------------------------------------------------------ + +#Ensure security conf directory is created +- name: Ensure security conf directory exists + become: yes + file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }} + changed_when: False + when: es_enable_xpack and "security" in es_xpack_features diff --git a/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-xpack-activation.yml b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-xpack-activation.yml new file mode 100644 index 000000000..cd72d6a7d --- /dev/null +++ b/ansible/roles/log-es6/tasks/xpack/security/elasticsearch-xpack-activation.yml @@ -0,0 +1,37 @@ +--- + +- name: Activate ES license (without security authentication) + uri: + method: PUT + url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true" + body_format: json + body: "{{ es_xpack_license }}" + return_content: yes + register: license_activated + no_log: True + when: not "security" in es_xpack_features + failed_when: > + license_activated.status != 200 or + license_activated.json.license_status is not defined or + license_activated.json.license_status != 'valid' + +- name: Activate ES license (with security authentication) + uri: + method: PUT + url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true" + user: "{{es_api_basic_auth_username}}" + password: "{{es_api_basic_auth_password}}" + body_format: json + force_basic_auth: yes + body: "{{ es_xpack_license }}" + return_content: yes + register: license_activated + no_log: True + when: "'security' in es_xpack_features" + failed_when: > + license_activated.status != 200 or + license_activated.json.license_status is not defined or + license_activated.json.license_status != 'valid' + +- debug: + msg: "License: {{ license_activated }}" diff --git a/ansible/roles/log-es6/templates/elasticsearch.j2 b/ansible/roles/log-es6/templates/elasticsearch.j2 new file mode 100644 index 000000000..5bf5746f9 --- /dev/null +++ b/ansible/roles/log-es6/templates/elasticsearch.j2 @@ -0,0 +1,83 @@ +################################ +# Elasticsearch +################################ + +# Elasticsearch home directory +ES_HOME={{es_home}} + +# Elasticsearch Java path +#JAVA_HOME= + +# Elasticsearch configuration directory +CONF_DIR={{conf_dir}} +ES_PATH_CONF={{conf_dir}} + +# Elasticsearch data directory +DATA_DIR={{ data_dirs | array_to_str }} + +# Elasticsearch logs directory +LOG_DIR={{log_dir}} + +# Elasticsearch PID directory +PID_DIR={{pid_dir}} + +ES_JVM_OPTIONS={{conf_dir}}/jvm.options + +# Configure restart on package upgrade (true, every other setting will lead to not restarting) +#ES_RESTART_ON_UPGRADE=true + +# Path to the GC log file +#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log + +################################ +# Elasticsearch service +################################ + +# SysV init.d +# +# When executing the init script, this user will be used to run the elasticsearch service. +# The default value is 'elasticsearch' and is declared in the init.d file. +# Note that this setting is only used by the init script. If changed, make sure that +# the configured user can read and write into the data, work, plugins and log directories. +# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service +ES_USER={{es_user}} +ES_GROUP={{es_group}} + +# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process +ES_STARTUP_SLEEP_TIME=5 + +################################ +# System properties +################################ + +# Specifies the maximum file descriptor number that can be opened by this process +# When using Systemd, this setting is ignored and the LimitNOFILE defined in +# /usr/lib/systemd/system/elasticsearch.service takes precedence +{% if es_max_open_files is defined %} +#MAX_OPEN_FILES +MAX_OPEN_FILES={{es_max_open_files}} +{% endif %} + +# The maximum number of bytes of memory that may be locked into RAM +# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option +# in elasticsearch.yml +# When using Systemd, the LimitMEMLOCK property must be set +# in /usr/lib/systemd/system/elasticsearch.service +#MAX_LOCKED_MEMORY= +{% if m_lock_enabled %} +MAX_LOCKED_MEMORY=unlimited +{% endif %} + +# Maximum number of VMA (Virtual Memory Areas) a process can own +# When using Systemd, this setting is ignored and the 'vm.max_map_count' +# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf +#MAX_MAP_COUNT=262144 +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} + +# Specifies the maximum number of threads that can be started. +# Elasticsearch requires a minimum of 2048. +{% if es_max_threads is defined %} +MAX_THREADS={{ es_max_threads }} +{% endif %} diff --git a/ansible/roles/log-es6/templates/elasticsearch.repo b/ansible/roles/log-es6/templates/elasticsearch.repo new file mode 100644 index 000000000..b6299042b --- /dev/null +++ b/ansible/roles/log-es6/templates/elasticsearch.repo @@ -0,0 +1,11 @@ +[elasticsearch-{{ es_major_version }}] +name=Elasticsearch repository for {{ es_major_version }} packages +baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=1 +autorefresh=1 +type=rpm-md +{% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %} +proxy=http://{{ es_proxy_host }}:{{es_proxy_port}} +{% endif %} diff --git a/ansible/roles/log-es6/templates/elasticsearch.yml.j2 b/ansible/roles/log-es6/templates/elasticsearch.yml.j2 new file mode 100644 index 000000000..ced4c41a1 --- /dev/null +++ b/ansible/roles/log-es6/templates/elasticsearch.yml.j2 @@ -0,0 +1,58 @@ + +{% if es_config %} +{{ es_config | to_nice_yaml }} +{% endif %} + +{% if es_config['cluster.name'] is not defined %} +cluster.name: elasticsearch +{% endif %} + +{% if (groups['log-es']|length) <= 2 %} +discovery.zen.minimum_master_nodes: 1 +{% else %} +discovery.zen.minimum_master_nodes: "{{ ((groups['log-es']|length) / 2 +1) | round(0, 'floor') | int}}" +{% endif %} + +{% if es_config['node.name'] is not defined %} +node.name: {{inventory_hostname}}-{{es_instance_name}} +{% endif %} + +#################################### Paths #################################### + +# Path to directory containing configuration (this file and logging.yml): + +{% if (es_version | version_compare('6.0.0', '<')) %} +path.conf: {{ conf_dir }} +{% endif %} + +path.data: {{ data_dirs | array_to_str }} + +path.logs: {{ log_dir }} + +{% if es_enable_xpack %} +{% if not "security" in es_xpack_features %} +xpack.security.enabled: false +{% endif %} + +{% if not "monitoring" in es_xpack_features %} +xpack.monitoring.enabled: false +{% endif %} + +{% if not "alerting" in es_xpack_features %} +xpack.watcher.enabled: false +{% endif %} + +{% if not "ml" in es_xpack_features %} +xpack.ml.enabled: false +{% endif %} + +{% if not "graph" in es_xpack_features %} +xpack.graph.enabled: false +{% endif %} +{% endif %} + +network.host: 0.0.0.0 + +{% if es_remote_reindex is defined %} +reindex.remote.whitelist: {{es_remote_host}}:9200 +{% endif %} diff --git a/ansible/roles/log-es6/templates/environment.j2 b/ansible/roles/log-es6/templates/environment.j2 new file mode 100644 index 000000000..7731297e1 --- /dev/null +++ b/ansible/roles/log-es6/templates/environment.j2 @@ -0,0 +1,2 @@ +JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/ +ES_PATH_CONF=/etc/elasticsearch/{{node_name}}/ diff --git a/ansible/roles/log-es6/templates/init/debian/elasticsearch.j2 b/ansible/roles/log-es6/templates/init/debian/elasticsearch.j2 new file mode 100755 index 000000000..efe2c37f6 --- /dev/null +++ b/ansible/roles/log-es6/templates/init/debian/elasticsearch.j2 @@ -0,0 +1,229 @@ +#!/bin/bash +# +# /etc/init.d/elasticsearch -- startup script for Elasticsearch +# +### BEGIN INIT INFO +# Provides: elasticsearch +# Required-Start: $network $remote_fs $named +# Required-Stop: $network $remote_fs $named +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts elasticsearch +# Description: Starts elasticsearch using start-stop-daemon +### END INIT INFO + +PATH=/bin:/usr/bin:/sbin:/usr/sbin +NAME={{es_instance_name}}_{{default_file | basename}} +{% if es_config['node.name'] is defined %} +DESC="Elasticsearch Server - {{es_config['node.name']}}" +{% else %} +DESC="Elasticsearch Server - {{es_instance_name}}" +{% endif %} + +DEFAULT=/etc/default/$NAME + +if [ `id -u` -ne 0 ]; then + echo "You need root privileges to run this script" + exit 1 +fi + +. /lib/lsb/init-functions +if [ -r /etc/default/rcS ]; then + . /etc/default/rcS +fi + +# The following variables can be overwritten in $DEFAULT + +# Run Elasticsearch as this user ID and group ID +ES_USER={{es_user}} +ES_GROUP={{es_group}} + +# Directory where the Elasticsearch binary distribution resides +ES_HOME={{es_home}} + +# Maximum number of open files +{% if es_max_open_files is defined %} +MAX_OPEN_FILES={{es_max_open_files}} +{% endif %} + +# Maximum amount of locked memory +#MAX_LOCKED_MEMORY= +{% if m_lock_enabled %} +MAX_LOCKED_MEMORY=unlimited +{% endif %} + +# Elasticsearch log directory +LOG_DIR={{log_dir}} + +# Elasticsearch data directory +DATA_DIR={{ data_dirs | array_to_str }} + +# Elasticsearch configuration directory +CONF_DIR={{conf_dir}} +ES_PATH_CONF={{ conf_dir }} + +# Maximum number of VMA (Virtual Memory Areas) a process can own +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} + +# Elasticsearch PID file directory +PID_DIR={{pid_dir}} + +ES_JVM_OPTIONS="{{conf_dir}}/jvm.options" + +# End of variables that can be overwritten in $DEFAULT + +# overwrite settings from default file +if [ -f "$DEFAULT" ]; then + . "$DEFAULT" +fi + +# CONF_FILE setting was removed +if [ ! -z "$CONF_FILE" ]; then + echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." + exit 1 +fi + +if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then + echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" +fi + +# Define other required variables +PID_FILE="$PID_DIR/$NAME.pid" +DAEMON=$ES_HOME/bin/elasticsearch +{% if (es_version | version_compare('6.0.0', '<')) %} +DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR" +{% else %} +DAEMON_OPTS="-d -p $PID_FILE" +{% endif %} + +export ES_JAVA_OPTS +export JAVA_HOME +export ES_INCLUDE +export ES_JVM_OPTIONS +export ES_PATH_CONF + +# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported +if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi +if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi +if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi +if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi +if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi +if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi +if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi +if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi + +# Check DAEMON exists +if [ ! -x "$DAEMON" ]; then + echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON" + exit 1 +fi + +checkJava() { + if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA="$JAVA_HOME/bin/java" + else + JAVA=`which java` + fi + + if [ ! -x "$JAVA" ]; then + echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" + exit 1 + fi +} + +case "$1" in + start) + checkJava + + log_daemon_msg "Starting $DESC" + + pid=`pidofproc -p $PID_FILE elasticsearch` + if [ -n "$pid" ] ; then + log_begin_msg "Already running." + log_end_msg 0 + exit 0 + fi + + # Ensure that the PID_DIR exists (it is cleaned at OS startup time) + if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then + mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR" + fi + if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then + touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE" + fi + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + if [ -n "$MAX_LOCKED_MEMORY" ]; then + ulimit -l $MAX_LOCKED_MEMORY + fi + + if [ -n "$MAX_THREADS" ]; then + ulimit -u $MAX_THREADS + fi + + if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then + sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT + fi + + # Start Daemon + start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS + return=$? + if [ $return -eq 0 ]; then + i=0 + timeout={{es_debian_startup_timeout}} + # Wait for the process to be properly started before exiting + until { kill -0 `cat "$PID_FILE"`; } >/dev/null 2>&1 + do + sleep 1 + i=$(($i + 1)) + if [ $i -gt $timeout ]; then + log_end_msg 1 + exit 1 + fi + done + fi + log_end_msg $return + exit $return + ;; + stop) + log_daemon_msg "Stopping $DESC" + + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" \ + --user "$ES_USER" \ + --quiet \ + --retry forever/TERM/20 > /dev/null + if [ $? -eq 1 ]; then + log_progress_msg "$DESC is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop $DESC (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_progress_msg "(not running)" + fi + log_end_msg 0 + ;; + status) + status_of_proc -p $PID_FILE elasticsearch elasticsearch && exit 0 || exit $? + ;; + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + fi + $0 start + ;; + *) + log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}" + exit 1 + ;; +esac + +exit 0 diff --git a/ansible/roles/log-es6/templates/init/redhat/elasticsearch.j2 b/ansible/roles/log-es6/templates/init/redhat/elasticsearch.j2 new file mode 100755 index 000000000..c993e1436 --- /dev/null +++ b/ansible/roles/log-es6/templates/init/redhat/elasticsearch.j2 @@ -0,0 +1,217 @@ +#!/bin/bash +# +# elasticsearch +# +# chkconfig: 2345 80 20 +# description: Starts and stops a single elasticsearch instance on this system +# + +### BEGIN INIT INFO +# Provides: Elasticsearch +# Required-Start: $network $named +# Required-Stop: $network $named +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: This service manages the elasticsearch daemon +# Description: Elasticsearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search. +### END INIT INFO + +# +# init.d / servicectl compatibility (openSUSE) +# +if [ -f /etc/rc.status ]; then + . /etc/rc.status + rc_reset +fi + +# +# Source function library. +# +if [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +fi + +# Sets the default values for elasticsearch variables used in this script +ES_USER="{{es_user}}" +ES_GROUP="{{es_group}}" +ES_HOME="{{es_home}}" +{% if es_max_open_files is defined %} +MAX_OPEN_FILES={{es_max_open_files}} +{% endif %} +# Maximum number of VMA (Virtual Memory Areas) a process can own +{% if es_max_map_count is defined %} +MAX_MAP_COUNT={{es_max_map_count}} +{% endif %} + +LOG_DIR="{{log_dir}}" +DATA_DIR={{ data_dirs | array_to_str }} +CONF_DIR="{{conf_dir}}" +ES_PATH_CONF="{{ conf_dir }}" + +PID_DIR="{{pid_dir}}" + +# Source the default env file +ES_ENV_FILE="{{instance_default_file}}" +if [ -f "$ES_ENV_FILE" ]; then + . "$ES_ENV_FILE" +fi + +if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then + echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]" +fi + +# CONF_FILE setting was removed +if [ ! -z "$CONF_FILE" ]; then + echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed." + exit 1 +fi + +exec="$ES_HOME/bin/elasticsearch" +prog="{{es_instance_name}}_{{default_file | basename}}" +pidfile="$PID_DIR/${prog}.pid" + +export ES_JAVA_OPTS +export JAVA_HOME +export ES_INCLUDE +export ES_JVM_OPTIONS +export ES_STARTUP_SLEEP_TIME +export ES_PATH_CONF + +# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported +if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi +if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi +if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi +if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi +if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi +if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi +if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi +if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi + +lockfile=/var/lock/subsys/$prog + +# backwards compatibility for old config sysconfig files, pre 0.90.1 +if [ -n $USER ] && [ -z $ES_USER ] ; then + ES_USER=$USER +fi + +if [ ! -x "$exec" ]; then + echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec" + exit 1 +fi + +checkJava() { + if [ -x "$JAVA_HOME/bin/java" ]; then + JAVA="$JAVA_HOME/bin/java" + else + JAVA=`which java` + fi + + if [ ! -x "$JAVA" ]; then + echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME" + exit 1 + fi +} + +start() { + checkJava + [ -x $exec ] || exit 5 + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + if [ -n "$MAX_LOCKED_MEMORY" ]; then + ulimit -l $MAX_LOCKED_MEMORY + fi + if [ -n "$MAX_THREADS" ]; then + ulimit -u $MAX_THREADS + fi + if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then + sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT + fi + + # Ensure that the PID_DIR exists (it is cleaned at OS startup time) + if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then + mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR" + fi + if [ -n "$pidfile" ] && [ ! -e "$pidfile" ]; then + touch "$pidfile" && chown "$ES_USER":"$ES_GROUP" "$pidfile" + fi + + cd $ES_HOME + echo -n $"Starting $prog: " + # if not running, start it up here, usually something like "daemon $exec" +{% if (es_version | version_compare('6.0.0', '<')) %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR +{% else %} + daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d +{% endif %} + retval=$? + echo + [ $retval -eq 0 ] && touch $lockfile + return $retval +} + +stop() { + echo -n $"Stopping $prog: " + # stop it here, often "killproc $prog" + killproc -p $pidfile -d 86400 $prog + retval=$? + echo + [ $retval -eq 0 ] && rm -f $lockfile + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $pidfile $prog +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac +exit $? diff --git a/ansible/roles/log-es6/templates/jvm.options.j2 b/ansible/roles/log-es6/templates/jvm.options.j2 new file mode 100644 index 000000000..853826ea3 --- /dev/null +++ b/ansible/roles/log-es6/templates/jvm.options.j2 @@ -0,0 +1,114 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms{{ log_es_heap_size | d('2g') }} +-Xmx{{ log_es_heap_size | d('2g') }} + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +-XX:+UseConcMarkSweepGC +-XX:CMSInitiatingOccupancyFraction=75 +-XX:+UseCMSInitiatingOccupancyOnly + +## optimizations + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# force the server VM +-server + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# use old-style file permissions on JDK9 +-Djdk.io.permissionsUseCanonicalPath=true + +# flags to configure Netty +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true +-Dlog4j.skipJansi=true + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps +# ensure the directory exists and has sufficient space +#-XX:HeapDumpPath=${heap.dump.path} + +## GC logging + +#-XX:+PrintGCDetails +#-XX:+PrintGCTimeStamps +#-XX:+PrintGCDateStamps +#-XX:+PrintClassHistogram +#-XX:+PrintTenuringDistribution +#-XX:+PrintGCApplicationStoppedTime + +# log GC status to a file with time stamps +# ensure the directory exists +#-Xloggc:${loggc} + + +# By default, the GC log file will not rotate. +# By uncommenting the lines below, the GC log file +# will be rotated every 128MB at most 32 times. +#-XX:+UseGCLogFileRotation +#-XX:NumberOfGCLogFiles=32 +#-XX:GCLogFileSize=128M + +# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON. +# If documents were already indexed with unquoted fields in a previous version +# of Elasticsearch, some operations may throw errors. +# +# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided +# only for migration purposes. +#-Delasticsearch.json.allow_unquoted_field_names=true +{% if es_jvm_custom_parameters !='' %} +{% for item in es_jvm_custom_parameters %} +{{ item }} +{% endfor %} +{% endif %} diff --git a/ansible/roles/log-es6/templates/log4j2.properties.j2 b/ansible/roles/log-es6/templates/log4j2.properties.j2 new file mode 100644 index 000000000..269be5290 --- /dev/null +++ b/ansible/roles/log-es6/templates/log4j2.properties.j2 @@ -0,0 +1,117 @@ +status = error + +# log action execution errors for easier debugging +logger.action.name = org.elasticsearch.action +logger.action.level = debug + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n + +appender.rolling.type = RollingFile +appender.rolling.name = rolling +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.rolling.fileName = ${sys:es.logs}.log +{% else %} +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +{% endif %} +appender.rolling.layout.type = PatternLayout +appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log +{% else %} +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +{% endif %} +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +{% if (es_version | version_compare('6.0.0', '>')) %} +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +{% endif %} +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling + +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log +{% else %} +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +{% endif %} +appender.deprecation_rolling.layout.type = PatternLayout +appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz +{% else %} +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz +{% endif %} +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 + +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.additivity = false + +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log +{% else %} +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log +{% endif %} +appender.index_search_slowlog_rolling.layout.type = PatternLayout +appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log +{% else %} +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log +{% endif %} +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.time.interval = 1 +appender.index_search_slowlog_rolling.policies.time.modulate = true + +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.additivity = false + +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log +{% else %} +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log +{% endif %} +appender.index_indexing_slowlog_rolling.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n +{% if (es_version | version_compare('6.0.0', '<')) %} +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +{% else %} +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log +{% endif %} +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.time.interval = 1 +appender.index_indexing_slowlog_rolling.policies.time.modulate = true + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.additivity = false diff --git a/ansible/roles/log-es6/templates/security/role_mapping.yml.j2 b/ansible/roles/log-es6/templates/security/role_mapping.yml.j2 new file mode 100644 index 000000000..258437522 --- /dev/null +++ b/ansible/roles/log-es6/templates/security/role_mapping.yml.j2 @@ -0,0 +1 @@ +{{ es_role_mapping | to_nice_yaml }} \ No newline at end of file diff --git a/ansible/roles/log-es6/templates/security/roles.yml.j2 b/ansible/roles/log-es6/templates/security/roles.yml.j2 new file mode 100644 index 000000000..9f211f2b0 --- /dev/null +++ b/ansible/roles/log-es6/templates/security/roles.yml.j2 @@ -0,0 +1 @@ +{{ es_roles.file | to_nice_yaml }} \ No newline at end of file diff --git a/ansible/roles/log-es6/templates/security/users_roles.j2 b/ansible/roles/log-es6/templates/security/users_roles.j2 new file mode 100644 index 000000000..1c0acfa1d --- /dev/null +++ b/ansible/roles/log-es6/templates/security/users_roles.j2 @@ -0,0 +1 @@ +{{users_roles | join("\n") }} \ No newline at end of file diff --git a/ansible/roles/log-es6/templates/systemd/elasticsearch.j2 b/ansible/roles/log-es6/templates/systemd/elasticsearch.j2 new file mode 100644 index 000000000..8bd5545c0 --- /dev/null +++ b/ansible/roles/log-es6/templates/systemd/elasticsearch.j2 @@ -0,0 +1,76 @@ +[Unit] +Description=Elasticsearch-{{es_instance_name}} +Documentation=http://www.elastic.co +Wants=network-online.target +After=network-online.target + +[Service] +Environment=ES_HOME={{es_home}} +Environment=CONF_DIR={{conf_dir}} +Environment=ES_PATH_CONF={{conf_dir}} +Environment=DATA_DIR={{ data_dirs | array_to_str }} +Environment=LOG_DIR={{log_dir}} +Environment=PID_DIR={{pid_dir}} +EnvironmentFile=-{{instance_default_file}} + +WorkingDirectory={{es_home}} + +User={{es_user}} +Group={{es_group}} + +{% if (es_version | version_compare('6.0.0', '<')) %} +ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec +{% endif %} + +ExecStart={{es_home}}/bin/elasticsearch \ + -p ${PID_DIR}/elasticsearch.pid \ +{% if (es_version | version_compare('6.0.0', '<')) %} + -Edefault.path.logs=${LOG_DIR} \ + -Edefault.path.data=${DATA_DIR} \ + -Edefault.path.conf=${CONF_DIR} \ +{% endif %} + --quiet + + +# StandardOutput is configured to redirect to journalctl since +# some error messages may be logged in standard output before +# elasticsearch logging system is initialized. Elasticsearch +# stores its logs in /var/log/elasticsearch and does not use +# journalctl by default. If you also want to enable journalctl +# logging, you can simply remove the "quiet" option from ExecStart. +StandardOutput=journal +StandardError=inherit + +# Specifies the maximum file descriptor number that can be opened by this process +{% if es_max_open_files is defined %} +LimitNOFILE={{es_max_open_files}} +{% endif %} + +# Specifies the maximum number of bytes of memory that may be locked into RAM +# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option +# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}} +{% if m_lock_enabled %} +LimitMEMLOCK=infinity +{% endif %} + +# Specifies the maximum number of threads that can be started. Elasticsearch requires a +# minimum of 2048. +LimitNPROC={{ es_max_threads }} + +# Disable timeout logic and wait until process is stopped +TimeoutStopSec=0 + +# SIGTERM signal is used to stop the Java process +KillSignal=SIGTERM + +# Send the signal only to the JVM rather than its control group +KillMode=process + +# Java process is never killed +SendSIGKILL=no + +# When a JVM receives a SIGTERM signal it exits with code 143 +SuccessExitStatus=143 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/log-es6/vars/Debian.yml b/ansible/roles/log-es6/vars/Debian.yml new file mode 100644 index 000000000..071736ef7 --- /dev/null +++ b/ansible/roles/log-es6/vars/Debian.yml @@ -0,0 +1,4 @@ +--- +java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}" +default_file: "/etc/default/elasticsearch" +es_home: "/usr/share/elasticsearch" diff --git a/ansible/roles/log-es6/vars/RedHat.yml b/ansible/roles/log-es6/vars/RedHat.yml new file mode 100644 index 000000000..b0aa42b2b --- /dev/null +++ b/ansible/roles/log-es6/vars/RedHat.yml @@ -0,0 +1,4 @@ +--- +java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}" +default_file: "/etc/sysconfig/elasticsearch" +es_home: "/usr/share/elasticsearch" \ No newline at end of file diff --git a/ansible/roles/log-es6/vars/main.yml b/ansible/roles/log-es6/vars/main.yml new file mode 100644 index 000000000..b3bd12671 --- /dev/null +++ b/ansible/roles/log-es6/vars/main.yml @@ -0,0 +1,8 @@ +--- +es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch" +es_conf_dir: "/etc/elasticsearch" +sysd_script: "/usr/lib/systemd/system/elasticsearch.service" +init_script: "/etc/init.d/elasticsearch" +#add supported features here +supported_xpack_features: ["alerting","monitoring","graph","security"] +reserved_xpack_users: ["elastic","kibana","logstash_system"] \ No newline at end of file diff --git a/ansible/roles/mobile_upload/defaults/main.yml b/ansible/roles/mobile_upload/defaults/main.yml new file mode 100644 index 000000000..49e165755 --- /dev/null +++ b/ansible/roles/mobile_upload/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for mobile_upload \ No newline at end of file diff --git a/ansible/roles/mobile_upload/tasks/main.yml b/ansible/roles/mobile_upload/tasks/main.yml new file mode 100644 index 000000000..4a5148e91 --- /dev/null +++ b/ansible/roles/mobile_upload/tasks/main.yml @@ -0,0 +1,13 @@ +--- +# tasks file for mobile_upload + - name: templating GDrive upload script + template: + src: "uploadToGdrive.sh" + dest: "./uploadToGdrive.sh" + - name: upload to {{ env_name }} + shell: find ../ -maxdepth 1 -iregex ".*[0-9]-debug.apk" -exec bash uploadToGdrive.sh -v -r {} \; + when: env_name=='production' + - name: upload to {{ env_name }} + shell: find ../ -maxdepth 1 -iregex ".*{{env_name}}-debug.apk" -exec bash uploadToGdrive.sh -v -r {} \; + when: env_name!='production' + diff --git a/ansible/roles/mobile_upload/templates/uploadToGdrive.sh b/ansible/roles/mobile_upload/templates/uploadToGdrive.sh new file mode 100755 index 000000000..90b23e8dd --- /dev/null +++ b/ansible/roles/mobile_upload/templates/uploadToGdrive.sh @@ -0,0 +1,343 @@ +#!/bin/bash + +# Upload a file to Google Drive +# Usage: upload.sh + +#!/bin/bashset -e +function usage(){ + + echo -e "\nThe script can be used to upload file/directory to google drive." + echo -e "\nUsage:\n $0 [options..] \n" + echo -e "Foldername argument is optional. If not provided, the file will be uploaded to preconfigured google drive. \n" + echo -e "File name argument is optional if create directory option is used. \n" + echo -e "Options:\n" + echo -e "-C | --create-dir - option to create directory. Will provide folder id." + echo -e "-r | --root-dir - google folder id to which the file/directory to upload." + echo -e "-v | --verbose - Display detailed message." + echo -e "-z | --config - Override default config file with custom config file." + echo -e "-h | --help - Display usage instructions.\n" + exit 0; +} + +file=${3} +#Configuration variables +ROOT_FOLDER="" +CLIENT_ID={{mobile_gdrive_client_id}} +CLIENT_SECRET={{mobile_gdrive_client_secret}} +API_REFRESH_TOKEN={{mobile_gdrive_refresh_token}} +FOLDER_ID={{mobile_gdrive_folder_id}} +FILE_NAME=${3} +SCOPE=${SCOPE:-"https://docs.google.com/feeds"} + +echo "FILE"${3} +#Internal variable +ACCESS_TOKEN="" +FILE="" +FOLDERNAME="" +curl_args="" + +DIR="$( cd "$( dirname "$( readlink "${BASH_SOURCE[0]}" )" )" && pwd )" + +if [ -e $HOME/.googledrive.conf ] +then + . $HOME/.googledrive.conf +fi + +PROGNAME=${0##*/} +SHORTOPTS="vhr:C:z:" +LONGOPTS="verbose,help,create-dir:,root-dir:,config:" + +set -o errexit -o noclobber -o pipefail #-o nounset +OPTS=$(getopt -s bash --options $SHORTOPTS --longoptions $LONGOPTS --name $PROGNAME -- "$@" ) + +# script to parse the input arguments +#if [ $? != 0 ] ; then echo "Failed parsing options." >&2 ; exit 1 ; fi + +eval set -- "$OPTS" + +VERBOSE=false +HELP=false +CONFIG="" +ROOTDIR="" + +while true; do + case "$1" in + -v | --verbose ) VERBOSE=true;curl_args="--progress"; shift ;; + -h | --help ) usage; shift ;; + -C | --create-dir ) FOLDERNAME="$2"; shift 2 ;; + -r | --root-dir ) ROOTDIR="$2";ROOT_FOLDER="$2"; shift 2 ;; + -z | --config ) CONFIG="$2"; shift 2 ;; + -- ) shift; break ;; + * ) break ;; + esac +done + +if [ ! -z "$CONFIG" ] + then + if [ -e "$CONFIG" ] + then + . $CONFIG + fi + if [ ! -z "$ROOTDIR" ] + then + ROOT_FOLDER="$ROOTDIR" + fi + +fi + +if [ ! -z "$1" ] +then + FILE=$1 +fi + +if [ ! -z "$2" ] && [ -z "$FOLDERNAME" ] +then + FOLDERNAME=$2 +fi + + +function log() { + + if [ "$VERBOSE" = true ]; then + echo -e "${1}" + + fi +} + +# Method to extract data from json response +function jsonValue() { +KEY=$1 +num=$2 +awk -F"[,:}][^://]" '{for(i=1;i<=NF;i++){if($i~/\042'$KEY'\042/){print $(i+1)}}}' | tr -d '"' | sed -n ${num}p | sed -e 's/[}]*$//' -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/[,]*$//' +} +# Method to upload files to google drive. Requires 3 arguments file path, google folder id and access token. +function uploadFile(){ + FILE="$1" + FOLDER_ID="$2" + ACCESS_TOKEN="$3" + SLUG=`basename "$FILE"` + FILENAME="${SLUG%.*}" + EXTENSION="${SLUG##*.}" + if [ "$FILENAME" == "$EXTENSION" -o ! "$(command -v mimetype)" ] + then + MIME_TYPE=`file --brief --mime-type "$FILE"` + else + MIME_TYPE=`mimetype --output-format %m "$FILE"` + + fi + + + FILESIZE=$(stat -c%s "$FILE") + + # JSON post data to specify the file name and folder under while the file to be created + postData="{\"mimeType\": \"$MIME_TYPE\",\"title\": \"$SLUG\",\"parents\": [{\"id\": \"${FOLDER_ID}\"}]}" + postDataSize=$(echo $postData | wc -c) + # Curl command to initiate resumable upload session and grab the location URL + log "Generating upload link for file $FILE ..." + uploadlink=`/usr/bin/curl \ + --silent \ + -X POST \ + -H "Host: www.googleapis.com" \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -H "X-Upload-Content-Type: $MIME_TYPE" \ + -H "X-Upload-Content-Length: $FILESIZE" \ + -d "$postData" \ + "https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ + --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n'` + + # Curl command to push the file to google drive. + # If the file size is large then the content can be split to chunks and uploaded. + # In that case content range needs to be specified. + log "Uploading file $FILE to google drive..." + curl \ + --silent \ + -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: $MIME_TYPE" \ + -H "Content-Length: $FILESIZE" \ + -H "Slug: $SLUG" \ + --upload-file "$FILE" \ + --output /dev/null \ + "$uploadlink" \ + $curl_args +} +if [ "$#" = "0" ] && [ -z "$FOLDERNAME" ] + then + echo "CLIENT_ID"${CLIENT_ID} + echo "CLIENT_SECRET"${CLIENT_SECRET} + echo "API_REFRESH_TOKEN"${API_REFRESH_TOKEN} + echo "FILE"${FILE_NAME} + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$API_REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` + echo "ACCESS_TOKEN"${ACCESS_TOKEN} + uploadFile "${FILE_NAME}" "$FOLDER_ID" "$ACCESS_TOKEN" + # usage +fi + + +# Method to create directory in google drive. Requires 3 arguments foldername,root directory id and access token. +function createDirectory(){ + DIRNAME="$1" + ROOTDIR="$2" + ACCESS_TOKEN="$3" + FOLDER_ID="" + QUERY="mimeType='application/vnd.google-apps.folder' and title='$DIRNAME'" + QUERY=$(echo $QUERY | sed -f ${DIR}/url_escape.sed) + + SEARCH_RESPONSE=`/usr/bin/curl \ + --silent \ + -XGET \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "https://www.googleapis.com/drive/v2/files/${ROOTDIR}/children?orderBy=title&q=${QUERY}&fields=items%2Fid"` + + FOLDER_ID=`echo $SEARCH_RESPONSE | jsonValue id` + + + if [ -z "$FOLDER_ID" ] + then + CREATE_FOLDER_POST_DATA="{\"mimeType\": \"application/vnd.google-apps.folder\",\"title\": \"$DIRNAME\",\"parents\": [{\"id\": \"$ROOTDIR\"}]}" + CREATE_FOLDER_RESPONSE=`/usr/bin/curl \ + --silent \ + -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H "Content-Type: application/json; charset=UTF-8" \ + -d "$CREATE_FOLDER_POST_DATA" \ + "https://www.googleapis.com/drive/v2/files?fields=id"` + FOLDER_ID=`echo $CREATE_FOLDER_RESPONSE | jsonValue id` + + fi +} + +# Method to upload files to google drive. Requires 3 arguments file path, google folder id and access token. +# function uploadFile(){ +# FILE="$1" +# FOLDER_ID="$2" +# ACCESS_TOKEN="$3" +# SLUG=`basename "$FILE"` +# FILENAME="${SLUG%.*}" +# EXTENSION="${SLUG##*.}" +# if [ "$FILENAME" == "$EXTENSION" -o ! "$(command -v mimetype)" ] +# then +# MIME_TYPE=`file --brief --mime-type "$FILE"` +# else +# MIME_TYPE=`mimetype --output-format %m "$FILE"` +# +# fi +# +# +# FILESIZE=$(stat -c%s "$FILE") +# +# # JSON post data to specify the file name and folder under while the file to be created +# postData="{\"mimeType\": \"$MIME_TYPE\",\"title\": \"$SLUG\",\"parents\": [{\"id\": \"${FOLDER_ID}\"}]}" +# postDataSize=$(echo $postData | wc -c) +# # Curl command to initiate resumable upload session and grab the location URL +# log "Generating upload link for file $FILE ..." +# uploadlink=`/usr/bin/curl \ +# --silent \ +# -X POST \ +# -H "Host: www.googleapis.com" \ +# -H "Authorization: Bearer ${ACCESS_TOKEN}" \ +# -H "Content-Type: application/json; charset=UTF-8" \ +# -H "X-Upload-Content-Type: $MIME_TYPE" \ +# -H "X-Upload-Content-Length: $FILESIZE" \ +# -d "$postData" \ +# "https://www.googleapis.com/upload/drive/v2/files?uploadType=resumable&supportsAllDrives=true&supportsTeamDrives=true" \ +# --dump-header - | sed -ne s/"Location: "//pi | tr -d '\r\n'` +# +# # Curl command to push the file to google drive. +# # If the file size is large then the content can be split to chunks and uploaded. +# # In that case content range needs to be specified. +# log "Uploading file $FILE to google drive..." +# curl \ +# --silent \ +# -X PUT \ +# -H "Authorization: Bearer ${ACCESS_TOKEN}" \ +# -H "Content-Type: $MIME_TYPE" \ +# -H "Content-Length: $FILESIZE" \ +# -H "Slug: $SLUG" \ +# --upload-file "$FILE" \ +# --output /dev/null \ +# "$uploadlink" \ +# $curl_args +# } + +old_umask=`umask` +umask 0077 + +if [ -z "$ROOT_FOLDER" ] +then + read -p "Root Folder ID (Default: root): " ROOT_FOLDER + if [ -z "$ROOT_FOLDER" ] || [ `echo $ROOT_FOLDER | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ] + then + ROOT_FOLDER="root" + echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf + else + if expr "$ROOT_FOLDER" : '^[A-Za-z0-9_-]\{28\}$' > /dev/null + then + echo "ROOT_FOLDER=$ROOT_FOLDER" >> $HOME/.googledrive.conf + else + echo "Invalid root folder id" + exit -1 + fi + fi +fi + +if [ -z "$CLIENT_ID" ] +then + read -p "Client ID: " CLIENT_ID + echo "CLIENT_ID=$CLIENT_ID" >> $HOME/.googledrive.conf +fi + +if [ -z "$CLIENT_SECRET" ] +then + read -p "Client Secret: " CLIENT_SECRET + echo "CLIENT_SECRET=$CLIENT_SECRET" >> $HOME/.googledrive.conf +fi + +if [ -z "$REFRESH_TOKEN" ] +then + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/device/code" --data "client_id=$CLIENT_ID&scope=$SCOPE"` + DEVICE_CODE=`echo "$RESPONSE" | jsonValue "device_code"` + USER_CODE=`echo "$RESPONSE" | jsonValue "user_code"` + URL=`echo "$RESPONSE" | jsonValue "verification_url"` + + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$API_REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo "$RESPONSE" | jsonValue access_token` +fi + +if [ -z "$ACCESS_TOKEN" ] + then + # Access token generation + RESPONSE=`curl --silent "https://accounts.google.com/o/oauth2/token" --data "client_id=$CLIENT_ID&client_secret=$CLIENT_SECRET&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token"` + ACCESS_TOKEN=`echo $RESPONSE | jsonValue access_token` +fi + +# Check to find whether the folder exists in google drive. If not then the folder is created in google drive under the configured root folder +if [ -z "$FOLDERNAME" ] || [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] + then + if [[ `echo "$FOLDERNAME" | tr [:upper:] [:lower:]` = `echo "root" | tr [:upper:] [:lower:]` ]] + then + ROOT_FOLDER="root" + fi + FOLDER_ID=$ROOT_FOLDER +else + FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` + +fi + log "Folder ID for folder name $FOLDERNAME : $FOLDER_ID" + +# Check whether the given file argument is valid and check whether the argument is file or directory. +# based on the type, if the argument is directory do a recursive upload. +if [ ! -z "$FILE" ]; then + if [ -f "$FILE" ];then + uploadFile "$FILE" "$FOLDER_ID" "$ACCESS_TOKEN" + elif [ -d "$FILE" ];then + FOLDERNAME=$(basename $FILE) + FOLDER_ID=`createDirectory "$FOLDERNAME" "$ROOT_FOLDER" "$ACCESS_TOKEN"` + for file in $(find "$FILE" -type f); + do + uploadFile "$file" "$FOLDER_ID" "$ACCESS_TOKEN" + done + fi +fi diff --git a/ansible/roles/mobile_upload/templates/url_escape.sed b/ansible/roles/mobile_upload/templates/url_escape.sed new file mode 100644 index 000000000..eac400265 --- /dev/null +++ b/ansible/roles/mobile_upload/templates/url_escape.sed @@ -0,0 +1,25 @@ +# sed url escaping +s:%:%25:g +s: :%20:g +s:<:%3C:g +s:>:%3E:g +s:#:%23:g +s:{:%7B:g +s:}:%7D:g +s:|:%7C:g +s:\\:%5C:g +s:\^:%5E:g +s:~:%7E:g +s:\[:%5B:g +s:\]:%5D:g +s:`:%60:g +s:;:%3B:g +s:/:%2F:g +s:?:%3F:g +s^:^%3A^g +s:@:%40:g +s:=:%3D:g +s:&:%26:g +s:\$:%24:g +s:\!:%21:g +s:\*:%2A:g diff --git a/ansible/roles/mongo-backup/tasks/backup_config.yml b/ansible/roles/mongo-backup/tasks/backup_config.yml deleted file mode 100644 index e789d666d..000000000 --- a/ansible/roles/mongo-backup/tasks/backup_config.yml +++ /dev/null @@ -1,55 +0,0 @@ -- name: copy the backup_config template - template: src=backup_config.j2 dest=/home/deployer/backup_config.js mode=0777 - - -- name: create backup directory for config - become: yes - file: path=/home/deployer/mongo-backups state=directory - -- name: create backup directory for config - become: yes - file: path=/home/deployer/mongo-backups/dump state=directory - -- name: backup backup_config for config - become: yes - shell: mongodump -o /home/deployer/mongo-backups/dump - ignore_errors: yes - -- name: zip the dump - become: yes - shell: zip -r "mongo_backup_{{type}}_`date +%Y%m%d`.zip" dump/ - args: - chdir: /home/deployer/mongo-backups - -- set_fact: - mongo_backup_gzip_file_name: "mongo_backup_{{type}}_{{ lookup('pipe', 'date +%Y%m%d') }}.zip" - - -- set_fact: - mongo_backup_gzip_file_path: "{{ mongo_backup_dir }}/{{ mongo_backup_gzip_file_name }}" - -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ mongo_backup_azure_container_name }} - ignore_errors: true - environment: - AZURE_STORAGE_ACCOUNT: "{{ mongo_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ mongo_backup_azure_storage_access_key }}" - -- name: Upload to azure blob storage - command: az storage blob upload --name {{ mongo_backup_gzip_file_name }} --file {{ mongo_backup_gzip_file_path }} --container-name {{ mongo_backup_azure_container_name }} - environment: - AZURE_STORAGE_ACCOUNT: "{{ mongo_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ mongo_backup_azure_storage_access_key }}" - async: 3600 - poll: 10 - -- name: clean up backup dir after upload - file: path="{{ mongo_backup_dir }}" state=absent - - -- name: run the template for backup config - become: yes - shell: mongo < /home/deployer/backup_config.js - async: 15 - poll: 0 - diff --git a/ansible/roles/mongo-backup/tasks/backup_replica.yml b/ansible/roles/mongo-backup/tasks/backup_replica.yml deleted file mode 100644 index 72fdcb1e4..000000000 --- a/ansible/roles/mongo-backup/tasks/backup_replica.yml +++ /dev/null @@ -1,55 +0,0 @@ -- name: copy the template backup_replica - template: src=backup_replica.j2 dest=/home/deployer/backup_replica.js mode=0777 - -- name: create backup directory - become: yes - file: path=/home/deployer/mongo-backups state=directory - -- name: create backup directory - become: yes - file: path=/home/deployer/mongo-backups/dump state=directory - -- name: backup backup_replica - become: yes - shell: mongodump -o /home/deployer/mongo-backups/dump - ignore_errors: yes - -- name: zip the dump - become: yes - shell: zip -r "mongo_backup_{{type}}_`date +%Y%m%d`.zip" dump/ - args: - chdir: /home/deployer/mongo-backups - -- set_fact: - mongo_backup_gzip_file_name: "mongo_backup_{{type}}_{{ lookup('pipe', 'date +%Y%m%d') }}.zip" - - -- set_fact: - mongo_backup_gzip_file_path: "{{ mongo_backup_dir }}/{{ mongo_backup_gzip_file_name }}" - -- name: Ensure azure blob storage container exists - command: az storage container create --name {{ mongo_backup_azure_container_name }} - ignore_errors: true - environment: - AZURE_STORAGE_ACCOUNT: "{{ mongo_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ mongo_backup_azure_storage_access_key }}" - -- name: Upload to azure blob storage - command: az storage blob upload --name {{ mongo_backup_gzip_file_name }} --file {{ mongo_backup_gzip_file_path }} --container-name {{ mongo_backup_azure_container_name }} - environment: - AZURE_STORAGE_ACCOUNT: "{{ mongo_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ mongo_backup_azure_storage_access_key }}" - async: 3600 - poll: 10 - -- name: clean up backup dir after upload - file: path="{{ mongo_backup_dir }}" state=absent - - -- name: run the template backup_replica - become: yes - shell: mongo < /home/deployer/backup_replica.js - async: 15 - poll: 0 - - diff --git a/ansible/roles/mongo-backup/tasks/lock_config.yml b/ansible/roles/mongo-backup/tasks/lock_config.yml deleted file mode 100644 index e050ce799..000000000 --- a/ansible/roles/mongo-backup/tasks/lock_config.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: copy the template lock_config - template: src=lock_config.j2 dest=/home/deployer/lock_config.js mode=0777 - -- name: run the template lock_config - become: yes - shell: mongo < /home/deployer/lock_config.js - async: 15 - poll: 0 - - diff --git a/ansible/roles/mongo-backup/tasks/lock_replica.yml b/ansible/roles/mongo-backup/tasks/lock_replica.yml deleted file mode 100644 index 7beed1f80..000000000 --- a/ansible/roles/mongo-backup/tasks/lock_replica.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: copy the template lock_shard - template: src=lock_shard.j2 dest=/home/deployer/lock_shard.js mode=0777 - -- name: run the template lock_shard - become: yes - shell: "mongo < /home/deployer/lock_shard.js" - async: 15 - poll: 0 - - diff --git a/ansible/roles/mongo-backup/tasks/main.yml b/ansible/roles/mongo-backup/tasks/main.yml deleted file mode 100644 index 594ccefb4..000000000 --- a/ansible/roles/mongo-backup/tasks/main.yml +++ /dev/null @@ -1,21 +0,0 @@ -- name: install zip - become: yes - apt: name=zip state=present - -- include: stop_query.yml - when: query is defined - -- include: lock_replica.yml - when: shard is defined - -- include: lock_config.yml - when: config is defined - -- include: backup_config.yml - when: config is defined - -- include: backup_replica.yml - when: shard is defined - -- include: start_query.yml - when: query is defined diff --git a/ansible/roles/mongo-backup/tasks/start_query.yml b/ansible/roles/mongo-backup/tasks/start_query.yml deleted file mode 100644 index 4686f91ee..000000000 --- a/ansible/roles/mongo-backup/tasks/start_query.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: copy the template start_balancer - template: src=start_balancer.j2 dest=/home/deployer/start_balancer.js mode=0777 - -- name: run the template start_balancer - become: yes - shell: mongo < /home/deployer/start_balancer.js - async: 15 - poll: 0 - diff --git a/ansible/roles/mongo-backup/tasks/stop_query.yml b/ansible/roles/mongo-backup/tasks/stop_query.yml deleted file mode 100644 index 409fdf5ad..000000000 --- a/ansible/roles/mongo-backup/tasks/stop_query.yml +++ /dev/null @@ -1,9 +0,0 @@ -- name: copy the template stop_balancer - template: src=stop_balancer.j2 dest=/home/deployer/stop_balancer.js mode=0777 - -- name: run the template stop_balancer - become: yes - shell: mongo < /home/deployer/stop_balancer.js - async: 15 - poll: 0 - diff --git a/ansible/roles/mongo-backup/templates/backup_config.j2 b/ansible/roles/mongo-backup/templates/backup_config.j2 deleted file mode 100644 index 9e6e3fe38..000000000 --- a/ansible/roles/mongo-backup/templates/backup_config.j2 +++ /dev/null @@ -1,2 +0,0 @@ -use config; -db.fsyncUnlock(); \ No newline at end of file diff --git a/ansible/roles/mongo-backup/templates/backup_replica.j2 b/ansible/roles/mongo-backup/templates/backup_replica.j2 deleted file mode 100644 index 78986821d..000000000 --- a/ansible/roles/mongo-backup/templates/backup_replica.j2 +++ /dev/null @@ -1 +0,0 @@ -db.fsyncUnlock(); diff --git a/ansible/roles/mongo-backup/templates/lock_config.j2 b/ansible/roles/mongo-backup/templates/lock_config.j2 deleted file mode 100644 index 445d30631..000000000 --- a/ansible/roles/mongo-backup/templates/lock_config.j2 +++ /dev/null @@ -1,2 +0,0 @@ -use config; -db.fsyncLock(); diff --git a/ansible/roles/mongo-backup/templates/lock_shard.j2 b/ansible/roles/mongo-backup/templates/lock_shard.j2 deleted file mode 100644 index 6e5a03609..000000000 --- a/ansible/roles/mongo-backup/templates/lock_shard.j2 +++ /dev/null @@ -1 +0,0 @@ -db.fsyncLock(); diff --git a/ansible/roles/mongo-backup/templates/start_balancer.j2 b/ansible/roles/mongo-backup/templates/start_balancer.j2 deleted file mode 100644 index 25abdd283..000000000 --- a/ansible/roles/mongo-backup/templates/start_balancer.j2 +++ /dev/null @@ -1,2 +0,0 @@ -use config; -sh.setBalancerState(true); diff --git a/ansible/roles/mongo-backup/templates/stop_balancer.j2 b/ansible/roles/mongo-backup/templates/stop_balancer.j2 deleted file mode 100644 index f26faa4c0..000000000 --- a/ansible/roles/mongo-backup/templates/stop_balancer.j2 +++ /dev/null @@ -1,2 +0,0 @@ -use config; -sh.stopBalancer(); \ No newline at end of file diff --git a/ansible/roles/mongodb-cluster/defaults/main.yml b/ansible/roles/mongodb-cluster/defaults/main.yml new file mode 100644 index 000000000..340a14c08 --- /dev/null +++ b/ansible/roles/mongodb-cluster/defaults/main.yml @@ -0,0 +1,135 @@ +--- + +mongodb_package: mongodb-org +mongodb_package_state: present +mongodb_version: "4.2" +mongodb_apt_keyserver: hkp://keyserver.ubuntu.com:80 +mongodb_apt_key_id: + "3.4": "0C49F3730359A14518585931BC711F9BA15703C6" + "3.6": "2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5" + "4.0": "9DA31620334BD75D9DCB49F368818C72E52529D4" + "4.2": "4B7C549A058F8B6B" + +mongodb_pymongo_from_pip: true # Install latest PyMongo via PIP or package manager +mongodb_pymongo_pip_version: 3.7.1 + +mongodb_user_update_password: "on_create" # MongoDB user password update default policy +mongodb_manage_service: true +mongodb_manage_systemd_unit: true +mongodb_systemd_unit_limit_nofile: 64000 +mongodb_systemd_unit_limit_nproc: 64000 + +mongodb_disable_transparent_hugepages: false + +mongodb_use_numa: true + +mongodb_user: "{{ 'mongod' if ('RedHat' == ansible_os_family) else 'mongodb' }}" +mongodb_uid: +mongodb_gid: +mongodb_daemon_name: "{{ 'mongod' if ('mongodb-org' in mongodb_package) else 'mongodb' }}" + +## net Options +mongodb_net_bindip: 0.0.0.0 # Comma separated list of ip addresses to listen on +mongodb_net_http_enabled: false # Enable http interface +mongodb_net_ipv6: false # Enable IPv6 support (disabled by default) +mongodb_net_maxconns: 65536 # Max number of simultaneous connections +mongodb_net_port: 27017 # Specify port number +mongodb_net_ssl: disabled # Enable or disable ssl connections +mongodb_net_ssl_mode: "" # Set the ssl mode (RequireSSL / preferSSL / AllowSSL / disabled) +mongodb_net_ssl_pemfile: "" # Location of the pemfile to use for ssl +mongodb_net_ssl_host: "" # Valid ssl hostname + +## processManagement Options +# Fork server process +# Enabled by default for RedHat as the init scripts assume forking is enabled. +mongodb_processmanagement_fork: false + +## security Options +# Disable or enable security. Possible values: 'disabled', 'enabled' +mongodb_security_authorization: "disabled" +mongodb_security_keyfile: /etc/mongodb-keyfile # Specify path to keyfile with password for inter-process authentication +mongodb_security_javascript_enabled: false + +## storage Options +mongodb_storage_dbpath: /data/db # Directory for datafiles +mongodb_storage_dirperdb: false # Use one directory per DB + +# The storage engine for the mongod database +mongodb_storage_engine: "wiredTiger" +# mmapv1 specific options +mongodb_storage_quota_enforced: false # Limits each database to a certain number of files +mongodb_storage_quota_maxfiles: 8 # Number of quota files per DB +mongodb_storage_smallfiles: false # Very useful for non-data nodes + +mongodb_storage_journal_enabled: true # Enable journaling +mongodb_storage_prealloc: true # Enable data file preallocation + +mongodb_wiredtiger_directory_for_indexes: false + +## systemLog Options +## The destination to which MongoDB sends all log output. Specify either 'file' or 'syslog'. +## If you specify 'file', you must also specify mongodb_systemlog_path. +mongodb_systemlog_destination: "file" +mongodb_systemlog_logappend: true # Append to logpath instead of over-writing +mongodb_systemlog_logrotate: "rename" # Logrotation behavior +mongodb_systemlog_path: /var/log/mongodb/{{ mongodb_daemon_name }}.log # Log file to send write to instead of stdout + +## operationProfiling Options +mongodb_operation_profiling_slow_op_threshold_ms: 100 +mongodb_operation_profiling_mode: "off" + +## cloud options (MongoDB >= 4.0) +mongodb_cloud_monitoring_free_state: "runtime" + +## replication Options +mongodb_replication_replset: "mongors" # Enable replication +mongodb_replication_replindexprefetch: "all" # specify index prefetching behavior (if secondary) [none|_id_only|all] +mongodb_replication_oplogsize: 1024 # specifies a maximum size in megabytes for the replication operation log + +# MMS Agent +mongodb_mms_agent_pkg: https://cloud.mongodb.com/download/agent/monitoring/mongodb-mms-monitoring-agent_7.2.0.488-1_amd64.ubuntu1604.deb +mongodb_mms_group_id: "" +mongodb_mms_api_key: "" +mongodb_mms_base_url: https://mms.mongodb.com + +#master configuration + +mongodb_login_host: "{{ groups['mongo_master'][0] }}" +host_name: "{{ groups['mongo_master'][0] }}" + +# password for inter-process authentication +# please regenerate this file on production environment with command 'openssl rand -base64 741' +mongodb_keyfile_content: | + 8pYcxvCqoe89kcp33KuTtKVf5MoHGEFjTnudrq5BosvWRoIxLowmdjrmUpVfAivh + CHjqM6w0zVBytAxH1lW+7teMYe6eDn2S/O/1YlRRiW57bWU3zjliW3VdguJar5i9 + Z+1a8lI+0S9pWynbv9+Ao0aXFjSJYVxAm/w7DJbVRGcPhsPmExiSBDw8szfQ8PAU + 2hwRl7nqPZZMMR+uQThg/zV9rOzHJmkqZtsO4UJSilG9euLCYrzW2hdoPuCrEDhu + Vsi5+nwAgYR9dP2oWkmGN1dwRe0ixSIM2UzFgpaXZaMOG6VztmFrlVXh8oFDRGM0 + cGrFHcnGF7oUGfWnI2Cekngk64dHA2qD7WxXPbQ/svn9EfTY5aPw5lXzKA87Ds8p + KHVFUYvmA6wVsxb/riGLwc+XZlb6M9gqHn1XSpsnYRjF6UzfRcRR2WyCxLZELaqu + iKxLKB5FYqMBH7Sqg3qBCtE53vZ7T1nefq5RFzmykviYP63Uhu/A2EQatrMnaFPl + TTG5CaPjob45CBSyMrheYRWKqxdWN93BTgiTW7p0U6RB0/OCUbsVX6IG3I9N8Uqt + l8Kc+7aOmtUqFkwo8w30prIOjStMrokxNsuK9KTUiPu2cj7gwYQ574vV3hQvQPAr + hhb9ohKr0zoPQt31iTj0FDkJzPepeuzqeq8F51HB56RZKpXdRTfY8G6OaOT68cV5 + vP1O6T/okFKrl41FQ3CyYN5eRHyRTK99zTytrjoP2EbtIZ18z+bg/angRHYNzbgk + lc3jpiGzs1ZWHD0nxOmHCMhU4usEcFbV6FlOxzlwrsEhHkeiununlCsNHatiDgzp + ZWLnP/mXKV992/Jhu0Z577DHlh+3JIYx0PceB9yzACJ8MNARHF7QpBkhtuGMGZpF + T+c73exupZFxItXs1Bnhe3djgE3MKKyYvxNUIbcTJoe7nhVMrwO/7lBSpVLvC4p3 + wR700U0LDaGGQpslGtiE56SemgoP + + +# names and passwords for administrative users +mongodb_user_admin_name: +mongodb_user_admin_password: + +mongodb_root_admin_name: +mongodb_root_admin_password: + +mongodb_root_backup_name: +mongodb_root_backup_password: + +# setParameter config +mongodb_set_parameters: + +# custom config options +mongodb_config: diff --git a/ansible/roles/mongodb-cluster/handlers/main.yml b/ansible/roles/mongodb-cluster/handlers/main.yml new file mode 100644 index 000000000..62251cf1c --- /dev/null +++ b/ansible/roles/mongodb-cluster/handlers/main.yml @@ -0,0 +1,44 @@ +--- + +- name: reload systemd + systemd: + daemon_reload: yes + +- name: run disable-transparent-hugepages + systemd: + name: disable-transparent-hugepages + enabled: true + state: started + daemon_reload: yes + +- name: mongodb restart + service: + name: "{{ mongodb_daemon_name }}" + state: restarted + when: mongodb_manage_service | bool + +- name: mongodb-mms-monitoring-agent restart + service: + name: mongodb-mms-monitoring-agent + state: restarted + when: mongodb_manage_service | bool + +- name: restart sysfsutils + service: + name: sysfsutils + state: restarted + +- name: wait when mongodb is started + wait_for: + host: "{{ item }}" + port: "{{ mongodb_net_port }}" + timeout: 120 + with_items: "{{ mongodb_net_bindip.split(',') | map('replace', '0.0.0.0', '127.0.0.1') | list }}" + +- name: wait when mongodb is started on localhost + wait_for: + host: "127.0.0.1" + port: "{{ mongodb_net_port }}" + delay: 5 + timeout: 120 + diff --git a/ansible/roles/mongodb-cluster/library/mongodb_replication.py b/ansible/roles/mongodb-cluster/library/mongodb_replication.py new file mode 100644 index 000000000..edc2253f7 --- /dev/null +++ b/ansible/roles/mongodb-cluster/library/mongodb_replication.py @@ -0,0 +1,466 @@ +#!/usr/bin/python + +# (c) 2015-2018, Sergei Antipov, 2GIS LLC +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +DOCUMENTATION = ''' +--- +module: mongodb_replication +short_description: Adds or removes a node from a MongoDB Replica Set. +description: + - Adds or removes host from a MongoDB replica set. Initialize replica set if it needed. +version_added: "2.4" +options: + login_user: + description: + - The username used to authenticate with + required: false + default: null + login_password: + description: + - The password used to authenticate with + required: false + default: null + login_host: + description: + - The host running the database + required: false + default: localhost + login_port: + description: + - The port to connect to + required: false + default: 27017 + login_database: + description: + - The database where login credentials are stored + required: false + default: admin + replica_set: + description: + - Replica set to connect to (automatically connects to primary for writes) + required: false + default: null + host_name: + description: + - The name of the host to add/remove from replica set + required: true + host_port: + description: + - The port of the host, which should be added/deleted from RS + required: true + default: null + host_type: + description: + - The type of the host in replica set + required: false + default: replica + choices: [ "replica", "arbiter" ] + ssl: + description: + - Whether to use an SSL connection when connecting to the database + default: False + ssl_cert_reqs: + description: + - Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided. + required: false + default: "CERT_REQUIRED" + choices: ["CERT_REQUIRED", "CERT_OPTIONAL", "CERT_NONE"] + build_indexes: + description: + - Determines whether the mongod builds indexes on this member. + required: false + default: true + hidden: + description: + - When this value is true, the replica set hides this instance, + and does not include the member in the output of db.isMaster() + or isMaster + required: false + default: false + priority: + description: + - A number that indicates the relative eligibility of a member + to become a primary + required: false + default: 1.0 + slave_delay: + description: + - The number of seconds behind the primary that this replica set + member should lag + required: false + default: 0 + votes: + description: + - The number of votes a server will cast in a replica set election + default: 1 + state: + state: + description: + - The replica set member state + required: false + default: present + choices: [ "present", "absent" ] +notes: + - Requires the pymongo Python package on the remote host, version 3.2+. It + can be installed using pip or the OS package manager. @see http://api.mongodb.org/python/current/installation.html +requirements: [ "pymongo" ] +author: "Sergei Antipov @UnderGreen" +''' + +EXAMPLES = ''' +# Add 'mongo1.dev:27017' host into replica set as replica (Replica will be initiated if it not exists) +- mongodb_replication: replica_set=replSet host_name=mongo1.dev host_port=27017 state=present + +# Add 'mongo2.dev:30000' host into replica set as arbiter +- mongodb_replication: replica_set=replSet host_name=mongo2.dev host_port=30000 host_type=arbiter state=present + +# Add 'mongo3.dev:27017' host into replica set as replica and authorization params +- mongodb_replication: replica_set=replSet login_host=mongo1.dev login_user=siteRootAdmin login_password=123456 host_name=mongo3.dev host_port=27017 state=present + +# Add 'mongo4.dev:27017' host into replica set as replica via SSL +- mongodb_replication: replica_set=replSet host_name=mongo4.dev host_port=27017 ssl=True state=present + +# Remove 'mongo4.dev:27017' host from the replica set +- mongodb_replication: replica_set=replSet host_name=mongo4.dev host_port=27017 state=absent +''' + +RETURN = ''' +host_name: + description: The name of the host to add/remove from replica set + returned: success + type: string + sample: "mongo3.dev" +host_port: + description: The port of the host, which should be added/deleted from RS + returned: success + type: int + sample: 27017 +host_type: + description: The type of the host in replica set + returned: success + type: string + sample: "replica" +''' +import ConfigParser +import ssl as ssl_lib +import time +from datetime import datetime as dtdatetime +from distutils.version import LooseVersion +try: + from pymongo.errors import ConnectionFailure + from pymongo.errors import OperationFailure + from pymongo.errors import ConfigurationError + from pymongo.errors import AutoReconnect + from pymongo.errors import ServerSelectionTimeoutError + from pymongo import version as PyMongoVersion + from pymongo import MongoClient +except ImportError: + pymongo_found = False +else: + pymongo_found = True + +# ========================================= +# MongoDB module specific support methods. +# +def check_compatibility(module, client): + srv_info = client.server_info() + if LooseVersion(PyMongoVersion) <= LooseVersion('3.2'): + module.fail_json(msg='Note: you must use pymongo 3.2+') + if LooseVersion(srv_info['version']) >= LooseVersion('3.4') and LooseVersion(PyMongoVersion) <= LooseVersion('3.4'): + module.fail_json(msg='Note: you must use pymongo 3.4+ with MongoDB 3.4.x') + if LooseVersion(srv_info['version']) >= LooseVersion('3.6') and LooseVersion(PyMongoVersion) <= LooseVersion('3.6'): + module.fail_json(msg='Note: you must use pymongo 3.6+ with MongoDB 3.6.x') + + +def check_members(state, module, client, host_name, host_port, host_type): + admin_db = client['admin'] + local_db = client['local'] + + if local_db.system.replset.count() > 1: + module.fail_json(msg='local.system.replset has unexpected contents') + + cfg = local_db.system.replset.find_one() + if not cfg: + module.fail_json(msg='no config object retrievable from local.system.replset') + + for member in cfg['members']: + if state == 'present': + if host_type == 'replica': + if "{0}:{1}".format(host_name, host_port) in member['host']: + module.exit_json(changed=False, host_name=host_name, host_port=host_port, host_type=host_type) + else: + if "{0}:{1}".format(host_name, host_port) in member['host'] and member['arbiterOnly']: + module.exit_json(changed=False, host_name=host_name, host_port=host_port, host_type=host_type) + else: + if host_type == 'replica': + if "{0}:{1}".format(host_name, host_port) not in member['host']: + module.exit_json(changed=False, host_name=host_name, host_port=host_port, host_type=host_type) + else: + if "{0}:{1}".format(host_name, host_port) not in member['host'] and member['arbiterOnly']: + module.exit_json(changed=False, host_name=host_name, host_port=host_port, host_type=host_type) + +def add_host(module, client, host_name, host_port, host_type, timeout=180, **kwargs): + start_time = dtdatetime.now() + while True: + try: + admin_db = client['admin'] + local_db = client['local'] + + if local_db.system.replset.count() > 1: + module.fail_json(msg='local.system.replset has unexpected contents') + + cfg = local_db.system.replset.find_one() + if not cfg: + module.fail_json(msg='no config object retrievable from local.system.replset') + + cfg['version'] += 1 + max_id = max(cfg['members'], key=lambda x:x['_id']) + new_host = { '_id': max_id['_id'] + 1, 'host': "{0}:{1}".format(host_name, host_port) } + if host_type == 'arbiter': + new_host['arbiterOnly'] = True + + if not kwargs['build_indexes']: + new_host['buildIndexes'] = False + + if kwargs['hidden']: + new_host['hidden'] = True + + if kwargs['priority'] != 1.0: + new_host['priority'] = kwargs['priority'] + + if kwargs['slave_delay'] != 0: + new_host['slaveDelay'] = kwargs['slave_delay'] + + if kwargs['votes'] != 1: + new_host['votes'] = kwargs['votes'] + + cfg['members'].append(new_host) + admin_db.command('replSetReconfig', cfg) + return + except (OperationFailure, AutoReconnect) as e: + if (dtdatetime.now() - start_time).seconds > timeout: + module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e)) + time.sleep(5) + +def remove_host(module, client, host_name, timeout=180): + start_time = dtdatetime.now() + while True: + try: + admin_db = client['admin'] + local_db = client['local'] + + if local_db.system.replset.count() > 1: + module.fail_json(msg='local.system.replset has unexpected contents') + + cfg = local_db.system.replset.find_one() + if not cfg: + module.fail_json(msg='no config object retrievable from local.system.replset') + + cfg['version'] += 1 + + if len(cfg['members']) == 1: + module.fail_json(msg="You can't delete last member of replica set") + for member in cfg['members']: + if host_name in member['host']: + cfg['members'].remove(member) + else: + fail_msg = "couldn't find member with hostname: {0} in replica set members list".format(host_name) + module.fail_json(msg=fail_msg) + except (OperationFailure, AutoReconnect) as e: + if (dtdatetime.now() - start_time).seconds > timeout: + module.fail_json(msg='reached timeout while waiting for rs.reconfig(): %s' % str(e)) + time.sleep(5) + +def load_mongocnf(): + config = ConfigParser.RawConfigParser() + mongocnf = os.path.expanduser('~/.mongodb.cnf') + + try: + config.readfp(open(mongocnf)) + creds = dict( + user=config.get('client', 'user'), + password=config.get('client', 'pass') + ) + except (ConfigParser.NoOptionError, IOError): + return False + + return creds + +def wait_for_ok_and_master(module, connection_params, timeout = 180): + start_time = dtdatetime.now() + while True: + try: + client = MongoClient(**connection_params) + authenticate(client, connection_params["username"], connection_params["password"]) + + status = client.admin.command('replSetGetStatus', check=False) + if status['ok'] == 1 and status['myState'] == 1: + return + + except ServerSelectionTimeoutError: + pass + + client.close() + + if (dtdatetime.now() - start_time).seconds > timeout: + module.fail_json(msg='reached timeout while waiting for rs.status() to become ok=1') + + time.sleep(1) + +def authenticate(client, login_user, login_password): + if login_user is None and login_password is None: + mongocnf_creds = load_mongocnf() + if mongocnf_creds is not False: + login_user = mongocnf_creds['user'] + login_password = mongocnf_creds['password'] + elif login_password is None and login_user is not None: + module.fail_json(msg='when supplying login arguments, both login_user and login_password must be provided') + + if login_user is not None and login_password is not None: + client.admin.authenticate(login_user, login_password) + +# ========================================= +# Module execution. +# + +def main(): + module = AnsibleModule( + argument_spec = dict( + login_user=dict(default=None), + login_password=dict(default=None, no_log=True), + login_host=dict(default='localhost'), + login_port=dict(default='27017'), + login_database=dict(default="admin"), + replica_set=dict(default=None), + host_name=dict(default='localhost'), + host_port=dict(default='27017'), + host_type=dict(default='replica', choices=['replica','arbiter']), + ssl=dict(default=False, type='bool'), + ssl_cert_reqs=dict(default='CERT_REQUIRED', choices=['CERT_NONE', 'CERT_OPTIONAL', 'CERT_REQUIRED']), + build_indexes = dict(type='bool', default='yes'), + hidden = dict(type='bool', default='no'), + priority = dict(default='1.0'), + slave_delay = dict(type='int', default='0'), + votes = dict(type='int', default='1'), + state=dict(default='present', choices=['absent', 'present']), + ) + ) + + if not pymongo_found: + module.fail_json(msg='the python pymongo (>= 3.2) module is required') + + login_user = module.params['login_user'] + login_password = module.params['login_password'] + login_host = module.params['login_host'] + login_port = module.params['login_port'] + login_database = module.params['login_database'] + replica_set = module.params['replica_set'] + host_name = module.params['host_name'] + host_port = module.params['host_port'] + host_type = module.params['host_type'] + ssl = module.params['ssl'] + state = module.params['state'] + priority = float(module.params['priority']) + + replica_set_created = False + + try: + if replica_set is None: + module.fail_json(msg='replica_set parameter is required') + else: + connection_params = { + "host": login_host, + "port": int(login_port), + "username": login_user, + "password": login_password, + "authsource": login_database, + "serverselectiontimeoutms": 5000, + "replicaset": replica_set, + } + + if ssl: + connection_params["ssl"] = ssl + connection_params["ssl_cert_reqs"] = getattr(ssl_lib, module.params['ssl_cert_reqs']) + + client = MongoClient(**connection_params) + authenticate(client, login_user, login_password) + client['admin'].command('replSetGetStatus') + + except ServerSelectionTimeoutError: + try: + connection_params = { + "host": login_host, + "port": int(login_port), + "username": login_user, + "password": login_password, + "authsource": login_database, + "serverselectiontimeoutms": 10000, + } + + if ssl: + connection_params["ssl"] = ssl + connection_params["ssl_cert_reqs"] = getattr(ssl_lib, module.params['ssl_cert_reqs']) + + client = MongoClient(**connection_params) + authenticate(client, login_user, login_password) + if state == 'present': + new_host = { '_id': 0, 'host': "{0}:{1}".format(host_name, host_port) } + if priority != 1.0: new_host['priority'] = priority + config = { '_id': "{0}".format(replica_set), 'members': [new_host] } + client['admin'].command('replSetInitiate', config) + client.close() + wait_for_ok_and_master(module, connection_params) + replica_set_created = True + module.exit_json(changed=True, host_name=host_name, host_port=host_port, host_type=host_type) + except OperationFailure as e: + module.fail_json(msg='Unable to initiate replica set: %s' % str(e)) + except ConnectionFailure as e: + module.fail_json(msg='unable to connect to database: %s' % str(e)) + + # reconnect again + client = MongoClient(**connection_params) + authenticate(client, login_user, login_password) + check_compatibility(module, client) + check_members(state, module, client, host_name, host_port, host_type) + + if state == 'present': + if host_name is None and not replica_set_created: + module.fail_json(msg='host_name parameter required when adding new host into replica set') + + try: + if not replica_set_created: + add_host(module, client, host_name, host_port, host_type, + build_indexes = module.params['build_indexes'], + hidden = module.params['hidden'], + priority = float(module.params['priority']), + slave_delay = module.params['slave_delay'], + votes = module.params['votes']) + except OperationFailure as e: + module.fail_json(msg='Unable to add new member to replica set: %s' % str(e)) + + elif state == 'absent': + try: + remove_host(module, client, host_name) + except OperationFailure as e: + module.fail_json(msg='Unable to remove member of replica set: %s' % str(e)) + + module.exit_json(changed=True, host_name=host_name, host_port=host_port, host_type=host_type) + +# import module snippets +from ansible.module_utils.basic import * +main() diff --git a/ansible/roles/mongodb-cluster/meta/main.yml b/ansible/roles/mongodb-cluster/meta/main.yml new file mode 100644 index 000000000..94bbca05a --- /dev/null +++ b/ansible/roles/mongodb-cluster/meta/main.yml @@ -0,0 +1,30 @@ +--- +dependencies: [] + +galaxy_info: + author: Sergei Antipov + company: Self-employed + description: Manage MongoDB with authentication and replica sets + license: GPLv2 + min_ansible_version: 2.5 + platforms: + - name: Ubuntu + versions: + - trusty + - xenial + - bionic + - name: Debian + versions: + - jessie + - stretch + - name: EL + versions: + - 6 + - 7 + - name: Amazon + versions: + - Candidate + galaxy_tags: + - database + - nosql + - mongodb diff --git a/ansible/roles/mongodb-cluster/tasks/auth_initialization.yml b/ansible/roles/mongodb-cluster/tasks/auth_initialization.yml new file mode 100644 index 000000000..033071ff3 --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/auth_initialization.yml @@ -0,0 +1,76 @@ +--- +- name: Use different mongod.conf for auth initialization + template: + src: mongod_init.conf.j2 + dest: /etc/mongod.conf + owner: root + group: root + mode: 0644 + notify: + - mongodb restart + - wait when mongodb is started on localhost + +- name: Flush all handlers at this point + meta: flush_handlers + +- name: create administrative user "{{ mongodb_user_admin_name }}" + mongodb_user: + database: admin + name: "{{ item.name }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: "{{ item.roles }}" + login_port: "{{ mongodb_net_port }}" + with_items: + - { + name: "{{ mongodb_user_admin_name }}", + password: "{{ mongodb_user_admin_password }}", + roles: "userAdminAnyDatabase" + } + no_log: true + +- name: create administrative user "{{ mongodb_root_admin_name }}" + mongodb_user: + database: admin + name: "{{ item.name }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: "{{ item.roles }}" + login_port: "{{ mongodb_net_port }}" + with_items: + - { + name: "{{ mongodb_root_admin_name }}", + password: "{{ mongodb_root_admin_password }}", + roles: "root" + } + no_log: true + +- name: create backup user "{{ mongodb_root_backup_name }}" + mongodb_user: + database: admin + name: "{{ item.name }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: "{{ item.roles }}" + login_port: "{{ mongodb_net_port }}" + with_items: + - { + name: "{{ mongodb_root_backup_name }}", + password: "{{ mongodb_root_backup_password }}", + roles: "backup,clusterMonitor" + } + no_log: true + +- name: Move back mongod.conf + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + owner: root + group: root + mode: 0644 + notify: + - mongodb restart + - wait when mongodb is started on localhost + +- name: Flush all handlers at this point + meta: flush_handlers diff --git a/ansible/roles/mongodb-cluster/tasks/configure.yml b/ansible/roles/mongodb-cluster/tasks/configure.yml new file mode 100644 index 000000000..5abfd2fea --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/configure.yml @@ -0,0 +1,87 @@ +--- +- name: create mongodb group + group: + name: "{{ mongodb_user }}" + state: present + +- name: create mongodb user + user: + name: "{{ mongodb_user }}" + group: "{{ mongodb_user }}" + state: present + +# Workaround because default(omit) is not working. At least in Ansible 2.8. + +- name: set mongodb gid + group: + name: "{{ mongodb_user }}" + gid: "{{ mongodb_gid }}" + state: present + when: mongodb_gid is defined + +- name: set mongodb uid + user: + name: "{{ mongodb_user }}" + uid: "{{ mongodb_uid }}" + group: "{{ mongodb_user }}" + state: present + when: mongodb_uid is defined + +- name: Create keyFile + copy: + dest: "{{ mongodb_security_keyfile }}" + content: "{{ mongodb_keyfile_content }}" + owner: "{{ mongodb_user }}" + group: "root" + mode: 0600 + when: mongodb_replication_replset | length > 0 + +- name: Create log dir if missing + file: + state: directory + dest: "{{ mongodb_systemlog_path|dirname }}" + owner: "{{ mongodb_user }}" + group: "{{ mongodb_user }}" + mode: 0755 + when: mongodb_systemlog_destination == "file" + +- name: Check than logfile exists + stat: path={{ mongodb_systemlog_path }} + register: logfile_stat + when: mongodb_systemlog_destination == "file" + +- name: Create log if missing + file: + state: touch + dest: "{{ mongodb_systemlog_path }}" + owner: "{{ mongodb_user }}" + group: "{{ mongodb_user }}" + mode: 0644 + when: ( mongodb_systemlog_destination == "file" + and logfile_stat is defined + and not logfile_stat.stat.exists ) + +- name: Ensure dbpath directory + file: + path: "{{ mongodb_storage_dbpath }}" + state: directory + owner: "{{ mongodb_user }}" + group: "{{ mongodb_user }}" + setype: 'mongod_var_lib_t' + recurse: true + follow: true + +- name: Configure mongodb + template: + src: mongod.conf.j2 + dest: /etc/mongod.conf + backup: true + owner: root + group: root + mode: 0644 + notify: + - mongodb restart + - wait when mongodb is started on localhost + +- name: Flush all handlers at this point + meta: flush_handlers diff --git a/ansible/roles/mongodb-cluster/tasks/disable_transparent_hugepages.yml b/ansible/roles/mongodb-cluster/tasks/disable_transparent_hugepages.yml new file mode 100644 index 000000000..654674ba7 --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/disable_transparent_hugepages.yml @@ -0,0 +1,16 @@ +--- +- name: Create disable transparent hugepages systemd service file + template: + src: "{{ item }}" + dest: /etc/systemd/system/disable-transparent-hugepages.service + owner: root + group: root + mode: 0644 + with_first_found: + - "disable-transparent-hugepages.{{ ansible_distribution | lower }}.service.j2" + - "disable-transparent-hugepages.{{ ansible_os_family | lower }}.service.j2" + notify: "run disable-transparent-hugepages" + when: + - ansible_service_mgr == "systemd" + - mongodb_disable_transparent_hugepages | bool + diff --git a/ansible/roles/mongodb-cluster/tasks/install.amazon.yml b/ansible/roles/mongodb-cluster/tasks/install.amazon.yml new file mode 100644 index 000000000..783298a4d --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/install.amazon.yml @@ -0,0 +1,49 @@ +--- +- name: Establish some role-related facts + set_fact: + mongodb_major_version: "{{ mongodb_version[0:3] }}" + +- name: Add YUM repository + template: + src: mongodb.repo.j2 + dest: /etc/yum.repos.d/mongodb.repo + mode: 0644 + with_items: "{{ mongodb_major_version }}" + loop_control: + loop_var: version_item + when: mongodb_package == 'mongodb-org' + +- name: Install MongoDB package + yum: + name: "{{ mongodb_package }}{% if (mongodb_version | length > 3) %}={{ mongodb_version }}{% endif %}" + state: "{{ mongodb_package_state }}" + lock_timeout: "{{ yum_lock_timeout }}" + +- name: Install numactl package + yum: + name: numactl + state: present + lock_timeout: "{{ yum_lock_timeout }}" + when: mongodb_use_numa | bool + +- name: Install PyMongo package + yum: + name: python-pymongo + state: present + lock_timeout: "{{ yum_lock_timeout }}" + when: not mongodb_pymongo_from_pip + +- name: Install PIP + yum: + name: + - python-devel + - python-pip + lock_timeout: "{{ yum_lock_timeout }}" + when: mongodb_pymongo_from_pip + +- name: Install PyMongo from PIP + pip: + name: pymongo + state: "{{ mongodb_pymongo_pip_version is defined | ternary('present', 'latest') }}" + version: "{{ mongodb_pymongo_pip_version | default(omit) }}" + when: mongodb_pymongo_from_pip diff --git a/ansible/roles/mongodb-cluster/tasks/install.debian.yml b/ansible/roles/mongodb-cluster/tasks/install.debian.yml new file mode 100644 index 000000000..9daf1a243 --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/install.debian.yml @@ -0,0 +1,113 @@ +--- +- name: Establish some role-related facts + set_fact: + mongodb_major_version: "{{ mongodb_version[0:3] }}" + +- name: Disable transparent huge pages on systemd systems + include_tasks: disable_transparent_hugepages.yml + when: + - mongodb_disable_transparent_hugepages | bool + - ansible_service_mgr == "systemd" + + #- name: Add APT key + # apt_key: + # keyserver: "{{ mongodb_apt_keyserver }}" + # id: "{{ mongodb_apt_key_id[mongodb_major_version] }}" + # when: mongodb_package == 'mongodb-org' + +- name: Add APT Key + become: yes + shell: "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 4B7C549A058F8B6B" + +- name: Fail when used wrong mongodb_version variable with Debian Stretch + fail: + msg: "mongodb_version variable should be '3.6' or '4.0' or '4.2' for Debian Stretch" + when: + - mongodb_package == 'mongodb-org' + - (mongodb_major_version != '3.6' and mongodb_major_version != '4.0' and mongodb_major_version != '4.2') + - ansible_distribution_release == 'stretch' + +- name: Fail when used wrong mongodb_version variable with Ubuntu 18.04 + fail: + msg: "mongodb_version variable should be '4.0' or '4.2', or else mongodb_package should be 'mongodb' for Ubuntu 18.04" + when: + - mongodb_package == 'mongodb-org' + - (mongodb_major_version != '4.0' and mongodb_major_version != '4.2') + - ansible_distribution_release == "bionic" + +- name: Fail when used wrong mongodb_version variable + fail: + msg: "mongodb_version variable should be '3.4', '3.6' or '4.0' or '4.2'" + when: (mongodb_package == 'mongodb-org' and + (mongodb_version is not defined + or mongodb_repository[mongodb_major_version] is not defined)) + +- name: Add APT repository + apt_repository: + repo: "{{ mongodb_repository[version_item] }}" + update_cache: true + with_items: "{{ mongodb_major_version }}" + loop_control: + loop_var: version_item + when: mongodb_package == 'mongodb-org' + +- name: Install MongoDB package + apt: + name: "{{ mongodb_package }}{% if (mongodb_version | length > 3) %}={{ mongodb_version }}{% endif %}" + state: "{{ mongodb_package_state }}" + update_cache: true + +- name: Install numactl package + apt: + name: numactl + state: present + when: mongodb_use_numa | bool + +- name: Add systemd configuration if present + template: + src: mongodb.service.j2 + dest: "/lib/systemd/system/{{mongodb_daemon_name}}.service" + owner: root + group: root + mode: '0644' + when: + - ansible_service_mgr == "systemd" + - mongodb_manage_systemd_unit | bool + notify: + - reload systemd + +- name: Add symlink for systemd + file: + src: "/lib/systemd/system/{{mongodb_daemon_name}}.service" + dest: "/etc/systemd/system/multi-user.target.wants/{{mongodb_daemon_name}}.service" + state: link + when: + - ansible_service_mgr == "systemd" + - mongodb_manage_systemd_unit | bool + notify: + - reload systemd + +- name: Install PyMongo package + apt: + name: python-pymongo + when: not mongodb_pymongo_from_pip + +- name: Install PIP + apt: + pkg: + - python-dev + - python-pip + when: mongodb_pymongo_from_pip | bool + +- name: Install setuptools (required for ansible 2.7+) + apt: + pkg: + - python-setuptools + when: mongodb_pymongo_from_pip | bool + +- name: Install PyMongo from PIP + pip: + name: pymongo + state: "{{ mongodb_pymongo_pip_version is defined | ternary('present', 'latest') }}" + version: "{{ mongodb_pymongo_pip_version | default(omit) }}" + when: mongodb_pymongo_from_pip | bool diff --git a/ansible/roles/mongodb-cluster/tasks/install.redhat.yml b/ansible/roles/mongodb-cluster/tasks/install.redhat.yml new file mode 100644 index 000000000..f2829181b --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/install.redhat.yml @@ -0,0 +1,55 @@ +--- + +- name: Establish some role-related facts + set_fact: + mongodb_major_version: "{{ mongodb_version[0:3] }}" + +- name: Install EPEL release repository + package: + name: epel-release + state: present + +- name: Add YUM repository + template: + src: mongodb.repo.j2 + dest: /etc/yum.repos.d/mongodb.repo + mode: 0644 + with_items: "{{ mongodb_major_version }}" + loop_control: + loop_var: version_item + when: mongodb_package == 'mongodb-org' + +- name: Install MongoDB package + yum: + name: "{{ mongodb_package }}{% if (mongodb_version | length > 3) %}={{ mongodb_version }}{% endif %}" + state: "{{ mongodb_package_state }}" + lock_timeout: "{{ yum_lock_timeout }}" + +- name: Install numactl package + yum: + name: numactl + state: present + lock_timeout: "{{ yum_lock_timeout }}" + when: mongodb_use_numa | bool + +- name: Install PyMongo package + yum: + name: python-pymongo + state: present + lock_timeout: "{{ yum_lock_timeout }}" + when: not mongodb_pymongo_from_pip + +- name: Install PIP + yum: + name: + - python-devel + - python-pip + lock_timeout: "{{ yum_lock_timeout }}" + when: mongodb_pymongo_from_pip | bool + +- name: Install PyMongo from PIP + pip: + name: pymongo + state: "{{ mongodb_pymongo_pip_version is defined | ternary('present', 'latest') }}" + version: "{{ mongodb_pymongo_pip_version | default(omit) }}" + when: mongodb_pymongo_from_pip | bool diff --git a/ansible/roles/mongodb-cluster/tasks/main.yml b/ansible/roles/mongodb-cluster/tasks/main.yml new file mode 100644 index 000000000..2ab22c9c1 --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/main.yml @@ -0,0 +1,137 @@ +--- + +- name: Check value of variable mongodb_net_ssl_host + fail: + msg: 'Set mongodb_net_ssl_mode is preferSSL or set valid hostname for mongodb_net_ssl_host!' + when: ( mongodb_net_ssl_mode == 'requireSSL' + and mongodb_net_ssl_host == '' ) + +- name: Check value of variable mongodb_login_host + fail: + msg: 'Set mongodb_login_host equal mongodb_net_ssl_host!' + when: ( mongodb_net_ssl_mode == 'requireSSL' + and mongodb_net_ssl_host != mongodb_login_host + and not mongodb_replication_replset ) + +- name: Include OS-specific variables + include_vars: "{{ item }}" + with_first_found: + - "{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution }}.yml" + - "{{ ansible_os_family }}.yml" + +- name: Include installation tasks + include: "{{ item }}" + with_first_found: + - "install.{{ ansible_distribution | lower }}.yml" + - "install.{{ ansible_os_family | lower }}.yml" + tags: [mongodb] + +- name: Include configuration.yml + include: configure.yml + tags: [mongodb] + +- name: Include replication and auth configuration + include: replication_init_auth.yml + when: ( mongodb_replication_replset | length > 0 + and mongodb_security_authorization == 'enabled' + and mongodb_master is defined and mongodb_master ) + tags: [mongodb] + +- name: Include replication configuration + include: replication.yml + when: mongodb_replication_replset | length > 0 + tags: [mongodb] + +- name: Check where admin user already exists + command: > + mongo --quiet {{ '--ssl --host ' + mongodb_net_ssl_host if mongodb_net_ssl_mode == 'requireSSL' else '' }} -u {{ mongodb_user_admin_name }} \ + -p {{ mongodb_user_admin_password }} --port {{ mongodb_net_port }} --eval 'db.version()' admin + register: mongodb_user_admin_check + changed_when: false + check_mode: no + ignore_errors: true + when: ( mongodb_security_authorization == 'enabled' + and not mongodb_replication_replset ) + no_log: true + tags: [mongodb] + +- name: Include authorization configuration + include: auth_initialization.yml + when: ( mongodb_security_authorization == 'enabled' + and not mongodb_replication_replset + and mongodb_user_admin_check.rc != 0 ) + tags: [mongodb] + +- name: create normal users with replicaset + mongodb_user: + database: "{{ item.database }}" + name: "{{ item.name }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: "{{ item.roles }}" + login_user: "{{ mongodb_user_admin_name }}" + login_password: "{{ mongodb_user_admin_password }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + with_items: + - "{{ mongodb_users | default([]) }}" + when: ( mongodb_replication_replset + and mongodb_security_authorization == 'enabled' + and mongodb_master is defined and mongodb_master ) + no_log: true + tags: [mongodb] + +- name: create normal users without replicaset + mongodb_user: + database: "{{ item.database }}" + name: "{{ item.name }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: "{{ item.roles }}" + login_user: "{{ mongodb_user_admin_name }}" + login_password: "{{ mongodb_user_admin_password }}" + login_port: "{{ mongodb_net_port }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + with_items: + - "{{ mongodb_users | default([]) }}" + when: ( mongodb_security_authorization == 'enabled' + and not mongodb_replication_replset ) + no_log: true + ignore_errors: true + tags: [mongodb] + +- name: create oplog user with replicaset + mongodb_user: + database: admin + user: "{{ item.user }}" + password: "{{ item.password }}" + update_password: "{{ mongodb_user_update_password }}" + roles: + - db: local + role: read + login_user: "{{ mongodb_user_admin_name }}" + login_password: "{{ mongodb_user_admin_password }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + with_items: + - "{{ mongodb_oplog_users | default([]) }}" + when: ( mongodb_replication_replset | length > 0 + and mongodb_security_authorization == 'enabled' + and mongodb_master is defined and mongodb_master ) + no_log: true + tags: [mongodb] + +- name: service started + service: + name: "{{ mongodb_daemon_name }}" + state: started + enabled: yes + +- name: Include MMS Agent configuration + include: mms-agent.yml + when: mongodb_mms_api_key | length > 0 + tags: [mongodb] diff --git a/ansible/roles/mongodb-cluster/tasks/mms-agent.yml b/ansible/roles/mongodb-cluster/tasks/mms-agent.yml new file mode 100644 index 000000000..84aba16ba --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/mms-agent.yml @@ -0,0 +1,32 @@ +--- +- name: Install MMS agent (Debian) + apt: + deb: "{{ mongodb_mms_agent_pkg }}" + when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' + +- name: Install MMS agent (RHEL) + yum: + name: "{{ mongodb_mms_agent_pkg }}" + state: present + when: ansible_distribution == 'CentOS' or ansible_distribution == 'RedHat' + +- name: Configure the MMS agent pt. 1 + file: + state: directory + path: /etc/mongodb-mms + owner: "{{ mongodb_user }}" + group: "{{ mongodb_user }}" + mode: 0755 + +- name: Configure the MMS agent pt. 2 + template: + src: monitoring-agent.config.j2 + dest: /etc/mongodb-mms/monitoring-agent.config + notify: mongodb-mms-monitoring-agent restart + +- name: Ensure that the MMS agent is started + service: + name: mongodb-mms-monitoring-agent + state: started + enabled: yes + diff --git a/ansible/roles/mongodb-cluster/tasks/replication.yml b/ansible/roles/mongodb-cluster/tasks/replication.yml new file mode 100644 index 000000000..3b5d106d1 --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/replication.yml @@ -0,0 +1,57 @@ +--- + +- name: Replication configuration + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + login_user: "{{ mongodb_root_admin_name }}" + login_password: "{{ mongodb_root_admin_password }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + when: mongodb_security_authorization == 'enabled' + with_items: + - "{{ mongodb_replication_params|default([]) }}" + +- name: Replication configuration without auth on master + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + when: ( mongodb_security_authorization == 'disabled' + and mongodb_master is defined and mongodb_master ) + with_items: + - "{{ mongodb_replication_params|default([]) }}" + +- name: Replication configuration without auth on replicas and arbiters + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_login_port|default(27017) }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + when: ( mongodb_security_authorization == 'disabled' + and ( mongodb_master is not defined or not mongodb_master) ) + with_items: + - "{{ mongodb_replication_params|default([]) }}" + diff --git a/ansible/roles/mongodb-cluster/tasks/replication_init_auth.yml b/ansible/roles/mongodb-cluster/tasks/replication_init_auth.yml new file mode 100644 index 000000000..359d5545e --- /dev/null +++ b/ansible/roles/mongodb-cluster/tasks/replication_init_auth.yml @@ -0,0 +1,42 @@ +--- +- name: Replication configuration | 1st Pt. + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_net_port|default(27017) }}" + login_user: "{{ mongodb_root_admin_name }}" + login_password: "{{ mongodb_root_admin_password }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + with_items: + - "{{ mongodb_replication_params|default([]) }}" + register: mongodb_replica_init + ignore_errors: true + +- include: auth_initialization.yml + when: mongodb_replica_init is failed + +- name: Replication configuration | 2nd Pt + mongodb_replication: + build_indexes: "{{ item.build_indexes|default(omit) }}" + login_host: "{{ mongodb_login_host|default('localhost') }}" + login_port: "{{ mongodb_net_port|default(27017) }}" + login_user: "{{ mongodb_root_admin_name }}" + login_password: "{{ mongodb_root_admin_password }}" + replica_set: "{{ mongodb_replication_replset }}" + host_name: "{{ host_name }}" + host_port: "{{ item.host_port|default(27017) }}" + host_type: "{{ item.host_type|default('replica') }}" + hidden: "{{ item.hidden|default(false) }}" + priority: "{{ item.priority|default(1.0) }}" + votes: "{{ item.votes|default(omit) }}" + ssl: "{{ True if mongodb_net_ssl_mode == 'requireSSL' else False }}" + when: mongodb_replica_init is failed + with_items: + - "{{ mongodb_replication_params|default([]) }}" diff --git a/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.amazon.service.j2 b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.amazon.service.j2 new file mode 100644 index 000000000..d728e1567 --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.amazon.service.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +Before={{ mongodb_daemon_name }}.service + +[Service] +Type=oneshot +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled' +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag' +ExecStart=-/bin/bash -c 'echo 0 > /sys/kernel/mm/redhat_transparent_hugepage/khugepaged/defrag' + +[Install] +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +RequiredBy={{ mongodb_daemon_name }}.service diff --git a/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.debian.service.j2 b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.debian.service.j2 new file mode 100644 index 000000000..b940261da --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.debian.service.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +Before={{ mongodb_daemon_name }}.service + +[Service] +Type=oneshot +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' +ExecStart=-/bin/bash -c 'echo 0 > /sys/kernel/mm/transparent_hugepage/khugepaged/defrag' + +[Install] +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +RequiredBy={{ mongodb_daemon_name }}.service diff --git a/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.redhat.service.j2 b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.redhat.service.j2 new file mode 100644 index 000000000..d728e1567 --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/disable-transparent-hugepages.redhat.service.j2 @@ -0,0 +1,17 @@ +# {{ ansible_managed }} +[Unit] +Description="Disable Transparent Hugepage before MongoDB boots" +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +Before={{ mongodb_daemon_name }}.service + +[Service] +Type=oneshot +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled' +ExecStart=-/bin/bash -c 'echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag' +ExecStart=-/bin/bash -c 'echo 0 > /sys/kernel/mm/redhat_transparent_hugepage/khugepaged/defrag' + +[Install] +#WARN: check service name on your system +# If you are using MongoDB Cloud, service name is "mongodb-mms-automation-agent.service" +RequiredBy={{ mongodb_daemon_name }}.service diff --git a/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 b/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 new file mode 100644 index 000000000..ebdead935 --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/mongod.conf.j2 @@ -0,0 +1,118 @@ +# {{ ansible_managed }} + +net: + bindIp: {{ mongodb_net_bindip }} + {% if mongodb_major_version is version("3.6", "<") -%} + http: + enabled: {{ mongodb_net_http_enabled | to_nice_json }} + {% endif -%} + ipv6: {{ mongodb_net_ipv6 | to_nice_json }} + maxIncomingConnections: {{ mongodb_net_maxconns }} + port: {{ mongodb_net_port }} + {% if mongodb_net_ssl_pemfile is defined and mongodb_net_ssl_mode is defined and mongodb_net_ssl == 'enabled' -%} + ssl: + mode: {{ mongodb_net_ssl_mode }} + PEMKeyFile: {{ mongodb_net_ssl_pemfile }} + {%- endif %} + {%- if mongodb_config['net'] is defined and mongodb_config['net'] is iterable %} + {%- for item in mongodb_config['net'] -%} + {{ item }} + {% endfor %} + {% endif %} + +processManagement: + fork: false + +{% if mongodb_replication_replset -%} +replication: + oplogSizeMB: {{ mongodb_replication_oplogsize | int }} + replSetName: {{ mongodb_replication_replset }} + {% if mongodb_storage_engine == 'mmapv1' -%} + secondaryIndexPrefetch: {{ mongodb_replication_replindexprefetch }} + {%- endif %} + {%- if mongodb_config['replication'] is defined and mongodb_config['replication'] is iterable %} + {%- for item in mongodb_config['replication'] -%} + {{ item }} + {% endfor %} + {% endif %} +{% endif %} + +security: + authorization: {{ mongodb_security_authorization }} + {% if mongodb_replication_replset and mongodb_security_authorization == 'enabled' -%} + keyFile: {{ mongodb_security_keyfile }} + {% endif -%} + javascriptEnabled: {{ mongodb_security_javascript_enabled | to_nice_json }} + {%- if mongodb_config['security'] is defined and mongodb_config['security'] is iterable %} + {%- for item in mongodb_config['security'] -%} + {{ item }} + {% endfor %} + {% endif %} + +storage: + dbPath: {{ mongodb_storage_dbpath }} + directoryPerDB: {{ mongodb_storage_dirperdb | to_nice_json }} + engine: {{ mongodb_storage_engine }} + journal: + enabled: {{ mongodb_storage_journal_enabled | to_nice_json }} + {% if mongodb_storage_engine == 'mmapv1' -%} + mmapv1: + quota: + enforced: {{ mongodb_storage_quota_enforced | to_nice_json }} + maxFilesPerDB: {{ mongodb_storage_quota_maxfiles }} + smallFiles: {{ mongodb_storage_smallfiles | to_nice_json }} + {% endif -%} + {% if mongodb_storage_engine == 'wiredTiger' -%} + wiredTiger: + engineConfig: + {% if mongodb_wiredtiger_cache_size is defined -%} + cacheSizeGB: {{ mongodb_wiredtiger_cache_size }} + {% endif -%} + directoryForIndexes: {{ mongodb_wiredtiger_directory_for_indexes | to_nice_json }} + {%- endif %} + {%- if mongodb_config['storage'] is defined and mongodb_config['storage'] is iterable %} + {%- for item in mongodb_config['storage'] -%} + {{ item }} + {% endfor %} + {% endif %} + +systemLog: + destination: {{ mongodb_systemlog_destination }} + {% if mongodb_systemlog_destination == 'file' -%} + logAppend: {{ mongodb_systemlog_logappend | to_nice_json }} + logRotate: {{ mongodb_systemlog_logrotate }} + path: {{ mongodb_systemlog_path }} + {%- endif %} + {%- if mongodb_config['systemLog'] is defined and mongodb_config['systemLog'] is iterable %} + {%- for item in mongodb_config['systemLog'] -%} + {{ item }} + {% endfor %} + {% endif %} + +operationProfiling: + slowOpThresholdMs: {{ mongodb_operation_profiling_slow_op_threshold_ms }} + mode: {{ mongodb_operation_profiling_mode }} + {%- if mongodb_config['operationProfiling'] is defined and mongodb_config['operationProfiling'] is iterable %} + {%- for item in mongodb_config['operationProfiling'] -%} + {{ item }} + {% endfor %} + {% endif %} + +{% if mongodb_major_version is version("4.0", ">=") -%} +cloud: + monitoring: + free: + state: {{ mongodb_cloud_monitoring_free_state }} + {%- if mongodb_config['cloud'] is defined and mongodb_config['cloud'] is iterable %} + {%- for item in mongodb_config['cloud'] -%} + {{ item }} + {% endfor %} + {% endif %} +{% endif %} + +{% if mongodb_set_parameters -%} +setParameter: + {% for key, value in mongodb_set_parameters.items() -%} + {{ key }}: {{ value }} + {% endfor %} +{% endif %} diff --git a/ansible/roles/mongodb-cluster/templates/mongod_init.conf.j2 b/ansible/roles/mongodb-cluster/templates/mongod_init.conf.j2 new file mode 100644 index 000000000..7d279d01a --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/mongod_init.conf.j2 @@ -0,0 +1,47 @@ +# {{ ansible_managed }} + +net: + bindIp: '127.0.0.1' + {% if mongodb_major_version is version("3.6", "<") -%} + http: + enabled: {{ mongodb_net_http_enabled | to_nice_json }} + {% endif -%} + ipv6: {{ mongodb_net_ipv6 | to_nice_json }} + maxIncomingConnections: {{ mongodb_net_maxconns }} + port: {{ mongodb_net_port }} + +processManagement: + fork: false + +security: + authorization: 'disabled' + +storage: + dbPath: {{ mongodb_storage_dbpath }} + directoryPerDB: {{ mongodb_storage_dirperdb | to_nice_json }} + engine: {{ mongodb_storage_engine }} + journal: + enabled: {{ mongodb_storage_journal_enabled | to_nice_json }} + {% if mongodb_storage_engine == 'mmapv1' -%} + mmapv1: + quota: + enforced: {{ mongodb_storage_quota_enforced | to_nice_json }} + maxFilesPerDB: {{ mongodb_storage_quota_maxfiles }} + smallFiles: {{ mongodb_storage_smallfiles | to_nice_json }} + {% endif -%} + {% if mongodb_storage_engine == 'wiredTiger' -%} + wiredTiger: + engineConfig: + {% if mongodb_wiredtiger_cache_size is defined -%} + cacheSizeGB: {{ mongodb_wiredtiger_cache_size }} + {% endif -%} + directoryForIndexes: {{ mongodb_wiredtiger_directory_for_indexes | to_nice_json }} + {% endif %} + +systemLog: + destination: {{ mongodb_systemlog_destination }} + {% if mongodb_systemlog_destination == 'file' -%} + logAppend: {{ mongodb_systemlog_logappend | to_nice_json }} + logRotate: {{ mongodb_systemlog_logrotate }} + path: {{ mongodb_systemlog_path }} + {% endif -%} diff --git a/ansible/roles/mongodb-cluster/templates/mongodb.repo.j2 b/ansible/roles/mongodb-cluster/templates/mongodb.repo.j2 new file mode 100644 index 000000000..727f2cb1b --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/mongodb.repo.j2 @@ -0,0 +1,10 @@ +[mongodb-org-{{ mongodb_major_version }}] +name=MongoDB {{ mongodb_major_version }} Repository +baseurl={{ mongodb_repository[version_item] }} +{% if mongodb_repository_gpgkey[version_item] is defined and mongodb_repository_gpgkey[version_item] != '' %} +gpgcheck=1 +gpgkey={{ mongodb_repository_gpgkey[version_item] }} +{% else %} +gpgcheck=0 +{% endif %} +enabled=1 diff --git a/ansible/roles/mongodb-cluster/templates/mongodb.service.j2 b/ansible/roles/mongodb-cluster/templates/mongodb.service.j2 new file mode 100644 index 000000000..61eb82c4c --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/mongodb.service.j2 @@ -0,0 +1,34 @@ +# {{ ansible_managed }} +[Unit] +Description=An object/document-oriented database +Documentation=man:mongod(1) +After=network-online.target + +[Service] +User={{ mongodb_user }} +{% if mongodb_use_numa | bool %} +ExecStart=/usr/bin/numactl --interleave=all /usr/bin/mongod --config /etc/mongod.conf +{% else %} +ExecStart=/usr/bin/mongod --config /etc/mongod.conf +{% endif %} +# file size +LimitFSIZE=infinity +# cpu time +LimitCPU=infinity +# virtual memory size +LimitAS=infinity +# open files +LimitNOFILE={{ mongodb_systemd_unit_limit_nofile }} +# processes/threads +LimitNPROC={{ mongodb_systemd_unit_limit_nproc }} +# locked memory +LimitMEMLOCK=infinity +# total threads (user+kernel) +TasksMax=infinity +TasksAccounting=false + +# Recommended limits for for mongod as specified in +# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/mongodb-cluster/templates/monitoring-agent.config.j2 b/ansible/roles/mongodb-cluster/templates/monitoring-agent.config.j2 new file mode 100644 index 000000000..2a50d10f4 --- /dev/null +++ b/ansible/roles/mongodb-cluster/templates/monitoring-agent.config.j2 @@ -0,0 +1,3 @@ +mmsApiKey={{ mongodb_mms_api_key }} +mmsBaseUrl={{ mongodb_mms_base_url }} +mmsGroupId={{ mongodb_mms_group_id }} diff --git a/ansible/roles/mongodb-cluster/vars/Amazon.yml b/ansible/roles/mongodb-cluster/vars/Amazon.yml new file mode 100644 index 000000000..1ad42a267 --- /dev/null +++ b/ansible/roles/mongodb-cluster/vars/Amazon.yml @@ -0,0 +1,15 @@ +--- +mongodb_repository: + "4.2": "https://repo.mongodb.org/yum/amazon/2/mongodb-org/4.2/x86_64/" + "4.0": "https://repo.mongodb.org/yum/amazon/2/mongodb-org/4.0/x86_64/" + "3.6": "https://repo.mongodb.org/yum/amazon/2013.03/mongodb-org/3.6/x86_64/" + "3.4": "https://repo.mongodb.org/yum/amazon/2013.03/mongodb-org/3.4/x86_64/" + +mongodb_repository_gpgkey: + "4.2": "https://www.mongodb.org/static/pgp/server-4.2.asc" + "4.0": "https://www.mongodb.org/static/pgp/server-4.0.asc" + "3.6": "https://www.mongodb.org/static/pgp/server-3.6.asc" + +mongodb_pidfile_path: "{{ '/var/run/mongodb/mongod.pid' if ('mongodb-org' in mongodb_package) else '' }}" + +yum_lock_timeout: 180 diff --git a/ansible/roles/mongodb-cluster/vars/Debian.yml b/ansible/roles/mongodb-cluster/vars/Debian.yml new file mode 100644 index 000000000..c4a956d0b --- /dev/null +++ b/ansible/roles/mongodb-cluster/vars/Debian.yml @@ -0,0 +1,6 @@ +--- +mongodb_repository: + "3.4": "deb http://repo.mongodb.org/apt/debian {{ ansible_distribution_release }}/mongodb-org/3.4 main" + "3.6": "deb http://repo.mongodb.org/apt/debian {{ ansible_distribution_release }}/mongodb-org/3.6 main" + "4.0": "deb http://repo.mongodb.org/apt/debian {{ ansible_distribution_release }}/mongodb-org/4.0 main" + "4.2": "deb http://repo.mongodb.org/apt/debian {{ ansible_distribution_release }}/mongodb-org/4.2 main" diff --git a/ansible/roles/mongodb-cluster/vars/RedHat.yml b/ansible/roles/mongodb-cluster/vars/RedHat.yml new file mode 100644 index 000000000..d04db94e3 --- /dev/null +++ b/ansible/roles/mongodb-cluster/vars/RedHat.yml @@ -0,0 +1,16 @@ +--- +mongodb_repository: + "3.4": "https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.4/$basearch/" + "3.6": "https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.6/$basearch/" + "4.0": "https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.0/$basearch/" + "4.2": "https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.2/$basearch/" + +mongodb_repository_gpgkey: + "3.4": "https://www.mongodb.org/static/pgp/server-3.4.asc" + "3.6": "https://www.mongodb.org/static/pgp/server-3.6.asc" + "4.0": "https://www.mongodb.org/static/pgp/server-4.0.asc" + "4.2": "https://www.mongodb.org/static/pgp/server-4.2.asc" + +mongodb_pidfile_path: "{{ '/var/run/mongodb/mongod.pid' if ('mongodb-org' in mongodb_package) else '' }}" + +yum_lock_timeout: 180 diff --git a/ansible/roles/mongodb-cluster/vars/Ubuntu.yml b/ansible/roles/mongodb-cluster/vars/Ubuntu.yml new file mode 100644 index 000000000..1054614c7 --- /dev/null +++ b/ansible/roles/mongodb-cluster/vars/Ubuntu.yml @@ -0,0 +1,6 @@ +--- +mongodb_repository: + "3.4": "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.4 multiverse" + "3.6": "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/3.6 multiverse" + "4.0": "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/4.0 multiverse" + "4.2": "deb http://repo.mongodb.org/apt/ubuntu {{ ansible_distribution_release }}/mongodb-org/4.2 multiverse" diff --git a/ansible/roles/mongodb/tasks/main.yml b/ansible/roles/mongodb/tasks/main.yml deleted file mode 100755 index abe84cb94..000000000 --- a/ansible/roles/mongodb/tasks/main.yml +++ /dev/null @@ -1,25 +0,0 @@ -- name: run the apt-key - become: yes - shell: 'apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6' - -- name: repo added - become: yes - shell: 'echo "deb [ arch=amd64,arm64 ] http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.4 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.4.list ' - -- name: apt update - become: yes - apt: update_cache=yes - -- name: install monngod - become: yes - apt: name=mongodb-org state=present - -- name: update template - become: yes - template: src=mongod.conf.j2 dest=/etc/mongod.conf mode=0755 - -- name: service start - become: yes - shell: mongod --config /etc/mongod.conf & - async: 10 - poll: 5 \ No newline at end of file diff --git a/ansible/roles/mongodb/templates/mongod.conf.j2 b/ansible/roles/mongodb/templates/mongod.conf.j2 deleted file mode 100755 index 7632d016e..000000000 --- a/ansible/roles/mongodb/templates/mongod.conf.j2 +++ /dev/null @@ -1,41 +0,0 @@ -# mongod.conf - -# for documentation of all options, see: -# http://docs.mongodb.org/manual/reference/configuration-options/ - -# Where and how to store data. -storage: - dbPath: /var/lib/mongodb - journal: - enabled: true -# engine: -# mmapv1: -# wiredTiger: - -# where to write logging data. -systemLog: - destination: file - logAppend: true - path: /var/log/mongodb/mongod.log - -# network interfaces -net: - port: 27017 - bindIp: 0.0.0.0 - - -#processManagement: - -#security: - -#operationProfiling: - -#replication: - -#sharding: - -## Enterprise-Only Options: - -#auditLog: - -#snmp: \ No newline at end of file diff --git a/ansible/roles/monit/defaults/main.yml b/ansible/roles/monit/defaults/main.yml index 39cf0cc54..170af870d 100755 --- a/ansible/roles/monit/defaults/main.yml +++ b/ansible/roles/monit/defaults/main.yml @@ -1,6 +1,8 @@ --- monit_version: 1:5.16-2 monit_slack_user: Monit - Alerter +monitor_alerts_slack_url: "" +monitor_alerts_slack_channel: "" monit_slack_webhook_url: "{{ monitor_alerts_slack_url }}" monit_slack_channel: "{{ monitor_alerts_slack_channel }}" diff --git a/ansible/roles/monit/templates/monitrc b/ansible/roles/monit/templates/monitrc index 0681b787c..79db90c41 100755 --- a/ansible/roles/monit/templates/monitrc +++ b/ansible/roles/monit/templates/monitrc @@ -1,43 +1,290 @@ -# Start Monit in the background (run as a daemon): - set daemon 60 # check services at 1-minute intervals - set logfile /var/log/monit.log - set idfile /var/lib/monit/id - set statefile /var/lib/monit/state +############################################################################### +## Monit control file +############################################################################### +## +## Comments begin with a '#' and extend through the end of the line. Keywords +## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. +## +## Below you will find examples of some frequently used statements. For +## information about the control file and a complete list of statements and +## options, please have a look in the Monit manual. +## +## +############################################################################### +## Global section +############################################################################### +## +## Start Monit in the background (run as a daemon): +# +set daemon 30 # check services at 30 seconds intervals +# with start delay 240 # optional: delay the first check by 4-minutes (by +# # default Monit check immediately after Monit start) +# +# +## Set syslog logging. If you want to log to a standalone log file instead, +## specify the full path to the log file +# +set logfile syslog - set mailserver {{ monitor_alerts_mail_server_host }} port {{ monitor_alerts_mail_server_port }} - username "{{ monitor_alerts_mail_server_username }}" - password "{{ monitor_alerts_mail_server_password }}" - using TLSV1 - with timeout 30 seconds +# +# +## Set the location of the Monit lock file which stores the process id of the +## running Monit instance. By default this file is stored in $HOME/.monit.pid +# +# set pidfile /var/run/monit.pid +# +## Set the location of the Monit id file which stores the unique id for the +## Monit instance. The id is generated and stored on first Monit start. By +## default the file is placed in $HOME/.monit.id. +# +# set idfile /var/.monit.id +# +## Set the location of the Monit state file which saves monitoring states +## on each cycle. By default the file is placed in $HOME/.monit.state. If +## the state file is stored on a persistent filesystem, Monit will recover +## the monitoring state across reboots. If it is on temporary filesystem, the +## state will be lost on reboot which may be convenient in some situations. +# +# set statefile /var/.monit.state +# +# - set httpd - port 2812 - allow 0.0.0.0/0 +## Set limits for various tests. The following example shows the default values: +## +# set limits { +# programOutput: 512 B, # check program's output truncate limit +# sendExpectBuffer: 256 B, # limit for send/expect protocol test +# fileContentBuffer: 512 B, # limit for file content test +# httpContentBuffer: 1 MB, # limit for HTTP content test +# networkTimeout: 5 seconds # timeout for network I/O +# } -## By default Monit will drop alert events if no mail servers are available. -## If you want to keep the alerts for later delivery retry, you can use the -## EVENTQUEUE statement. - set eventqueue - basedir /var/lib/monit/events # set the base directory where events will be stored - slots 100 # optionally limit the queue size - - set mail-format { - from: {{ monitor_alerts_mail_from_email }} - subject: [{{ env }}] monit alert -- $EVENT $SERVICE - message: $EVENT Service $SERVICE - Date: $DATE - Action: $ACTION - Host: $HOST - Description: $DESCRIPTION - - Your faithful employee, - Monit - } - -{% for mail_id in alerts_mailing_list.split(',') %} - set alert {{ mail_id }} not on { instance, pid, ppid } -{% endfor %} +## Set global SSL options (just most common options showed, see manual for +## full list). +# +# set ssl { +# verify : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED) +# selfsigned : allow # allow self signed SSL certificates (reject by default) +# } +# +# +## Set the list of mail servers for alert delivery. Multiple servers may be +## specified using a comma separator. If the first mail server fails, Monit +# will use the second mail server in the list and so on. By default Monit uses +# port 25 - it is possible to override this with the PORT option. +# +# set mailserver mail.bar.baz, # primary mailserver +# backup.bar.baz port 10025, # backup mailserver on port 10025 +# localhost # fallback relay +# +# +## By default Monit will drop alert events if no mail servers are available. +## If you want to keep the alerts for later delivery retry, you can use the +## EVENTQUEUE statement. The base directory where undelivered alerts will be +## stored is specified by the BASEDIR option. You can limit the queue size +## by using the SLOTS option (if omitted, the queue is limited by space +## available in the back end filesystem). +# +# set eventqueue +# basedir /var/monit # set the base directory where events will be stored +# slots 100 # optionally limit the queue size +# +# +## Send status and events to M/Monit (for more informations about M/Monit +## see http://mmonit.com/). By default Monit registers credentials with +## M/Monit so M/Monit can smoothly communicate back to Monit and you don't +## have to register Monit credentials manually in M/Monit. It is possible to +## disable credential registration using the commented out option below. +## Though, if safety is a concern we recommend instead using https when +## communicating with M/Monit and send credentials encrypted. +# +# set mmonit http://monit:monit@192.168.1.10:8080/collector +# # and register without credentials # Don't register credentials +# +# +## Monit by default uses the following format for alerts if the the mail-format +## statement is missing:: +## --8<-- +## set mail-format { +## from: Monit +## subject: monit alert -- $EVENT $SERVICE +## message: $EVENT Service $SERVICE +## Date: $DATE +## Action: $ACTION +## Host: $HOST +## Description: $DESCRIPTION +## +## Your faithful employee, +## Monit +## } +## --8<-- +## +## You can override this message format or parts of it, such as subject +## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. +## are expanded at runtime. For example, to override the sender, use: +# +# set mail-format { from: monit@foo.bar } +# +# +## You can set alert recipients whom will receive alerts if/when a +## service defined in this file has errors. Alerts may be restricted on +## events by using a filter as in the second example below. +# +# set alert sysadm@foo.bar # receive all alerts +# +## Do not alert when Monit starts, stops or performs a user initiated action. +## This filter is recommended to avoid getting alerts for trivial cases. +# +# set alert your-name@your.domain not on { instance, action } +# +# +## Monit has an embedded HTTP interface which can be used to view status of +## services monitored and manage services from a web interface. The HTTP +## interface is also required if you want to issue Monit commands from the +## command line, such as 'monit status' or 'monit restart service' The reason +## for this is that the Monit client uses the HTTP interface to send these +## commands to a running Monit daemon. See the Monit Wiki if you want to +## enable SSL for the HTTP interface. +# +set httpd port 2812 and + use address localhost # only accept connection from localhost + allow localhost # allow localhost to connect to the server and + allow admin:monit # require user 'admin' with password 'monit' +############################################################################### +## Services +############################################################################### +## +## Check general system resources such as load average, cpu and memory +## usage. Each test specifies a resource, conditions and the action to be +## performed should a test fail. +# +# check system $HOST +# if loadavg (1min) > 4 then alert +# if loadavg (5min) > 2 then alert +# if cpu usage > 95% for 10 cycles then alert +# if memory usage > 75% then alert +# if swap usage > 25% then alert +# +# +## Check if a file exists, checksum, permissions, uid and gid. In addition +## to alert recipients in the global section, customized alert can be sent to +## additional recipients by specifying a local alert handler. The service may +## be grouped using the GROUP option. More than one group can be specified by +## repeating the 'group name' statement. +# +# check file apache_bin with path /usr/local/apache/bin/httpd +# if failed checksum and +# expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor +# if failed permission 755 then unmonitor +# if failed uid root then unmonitor +# if failed gid root then unmonitor +# alert security@foo.bar on { +# checksum, permission, uid, gid, unmonitor +# } with the mail-format { subject: Alarm! } +# group server +# +# +## Check that a process is running, in this case Apache, and that it respond +## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory, +## and number of children. If the process is not running, Monit will restart +## it by default. In case the service is restarted very often and the +## problem remains, it is possible to disable monitoring using the TIMEOUT +## statement. This service depends on another service (apache_bin) which +## is defined above. +# +# check process apache with pidfile /usr/local/apache/logs/httpd.pid +# start program = "/etc/init.d/httpd start" with timeout 60 seconds +# stop program = "/etc/init.d/httpd stop" +# if cpu > 60% for 2 cycles then alert +# if cpu > 80% for 5 cycles then restart +# if totalmem > 200.0 MB for 5 cycles then restart +# if children > 250 then restart +# if loadavg(5min) greater than 10 for 8 cycles then stop +# if failed host www.tildeslash.com port 80 protocol http +# and request "/somefile.html" +# then restart +# if failed port 443 protocol https with timeout 15 seconds then restart +# if 3 restarts within 5 cycles then unmonitor +# depends on apache_bin +# group server +# +# +## Check filesystem permissions, uid, gid, space and inode usage. Other services, +## such as databases, may depend on this resource and an automatically graceful +## stop may be cascaded to them before the filesystem will become full and data +## lost. +# +# check filesystem datafs with path /dev/sdb1 +# start program = "/bin/mount /data" +# stop program = "/bin/umount /data" +# if failed permission 660 then unmonitor +# if failed uid root then unmonitor +# if failed gid disk then unmonitor +# if space usage > 80% for 5 times within 15 cycles then alert +# if space usage > 99% then stop +# if inode usage > 30000 then alert +# if inode usage > 99% then stop +# group server +# +# +## Check a file's timestamp. In this example, we test if a file is older +## than 15 minutes and assume something is wrong if its not updated. Also, +## if the file size exceed a given limit, execute a script +# +# check file database with path /data/mydatabase.db +# if failed permission 700 then alert +# if failed uid data then alert +# if failed gid data then alert +# if timestamp > 15 minutes then alert +# if size > 100 MB then exec "/my/cleanup/script" as uid dba and gid dba +# +# +## Check directory permission, uid and gid. An event is triggered if the +## directory does not belong to the user with uid 0 and gid 0. In addition, +## the permissions have to match the octal description of 755 (see chmod(1)). +# +# check directory bin with path /bin +# if failed permission 755 then unmonitor +# if failed uid 0 then unmonitor +# if failed gid 0 then unmonitor +# +# +## Check a remote host availability by issuing a ping test and check the +## content of a response from a web server. Up to three pings are sent and +## connection to a port and an application level network check is performed. +# +# check host myserver with address 192.168.1.1 +# if failed ping then alert +# if failed port 3306 protocol mysql with timeout 15 seconds then alert +# if failed port 80 protocol http +# and request /some/path with content = "a string" +# then alert +# +# +## Check a network link status (up/down), link capacity changes, saturation +## and bandwidth usage. +# +# check network public with interface eth0 +# if failed link then alert +# if changed link then alert +# if saturation > 90% then alert +# if download > 10 MB/s then alert +# if total upload > 1 GB in last hour then alert +# +# +## Check custom program status output. +# +# check program myscript with path /usr/local/bin/myscript.sh +# if status != 0 then alert +# +# +############################################################################### +## Includes +############################################################################### +## ## It is possible to include additional configuration parts from other files or ## directories. - include /etc/monit/conf.d/* +# +# include /etc/monit.d/* +# diff --git a/ansible/roles/offline-installer/tasks/uploadToAzure.yml b/ansible/roles/offline-installer/tasks/uploadToAzure.yml index aa06b7e6d..62d4378d1 100644 --- a/ansible/roles/offline-installer/tasks/uploadToAzure.yml +++ b/ansible/roles/offline-installer/tasks/uploadToAzure.yml @@ -1,11 +1,11 @@ - name: Ensure azure blob storage container exists command: az storage container create --name {{offline_installer_container_name}} --public-access blob environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_offline_azure_storage_account}}" - AZURE_STORAGE_KEY: "{{sunbird_offline_azure_storage_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" - name: Get the environment name for the artifact name - shell: "cat {{offline_repo_location}}/offline-installer-repo/src/env.json | jq -r '.APP_NAME'" + shell: "cat {{offline_repo_location}}/offline-installer-repo/src/package.json | jq -r '.name'" register: env_name - name: Display the environment name of the installer @@ -61,8 +61,8 @@ args: chdir: "{{offline_repo_location}}/offline-installer-repo/" environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_offline_azure_storage_account}}" - AZURE_STORAGE_KEY: "{{sunbird_offline_azure_storage_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" async: 60 poll: 10 @@ -71,8 +71,8 @@ args: chdir: "{{offline_repo_location}}/offline-installer-repo/offline_artifacts/" environment: - AZURE_STORAGE_ACCOUNT: "{{sunbird_offline_azure_storage_account}}" - AZURE_STORAGE_KEY: "{{sunbird_offline_azure_storage_key}}" + AZURE_STORAGE_ACCOUNT: "{{sunbird_public_storage_account_name}}" + AZURE_STORAGE_KEY: "{{sunbird_public_storage_account_key}}" async: 60 poll: 10 @@ -99,5 +99,4 @@ state: directory recurse: yes owner: jenkins - group: jenkins - + group: jenkins \ No newline at end of file diff --git a/ansible/roles/offline-installer/templates/artifacts.sh.j2 b/ansible/roles/offline-installer/templates/artifacts.sh.j2 index ed3fa51a1..ea5db269d 100644 --- a/ansible/roles/offline-installer/templates/artifacts.sh.j2 +++ b/ansible/roles/offline-installer/templates/artifacts.sh.j2 @@ -3,13 +3,13 @@ if [ "{{offline_installer_type}}" == "windows32bit" ]; then - cp '{{offline_repo_location}}/offline-installer-repo/src/dist/1.0.0/win/ia32/{{environment_name}} Setup 1.0.0.exe' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_windows32bit.exe + cp '{{offline_repo_location}}/offline-installer-repo/src/dist/{{installer_version}}/win/ia32/{{environment_name}} Setup {{installer_version}}.exe' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_windows32bit.exe elif [ "{{offline_installer_type}}" == "windows64bit" ]; then - cp '{{offline_repo_location}}/offline-installer-repo/src/dist/1.0.0/win/x64/{{environment_name}} Setup 1.0.0.exe' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_windows64bit.exe + cp '{{offline_repo_location}}/offline-installer-repo/src/dist/{{installer_version}}/win/x64/{{environment_name}} Setup {{installer_version}}.exe' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_windows64bit.exe elif [ "{{offline_installer_type}}" == "linux64bit" ]; then - cp '{{offline_repo_location}}/offline-installer-repo/src/dist/1.0.0/linux/x64/{{environment_name}}_1.0.0_amd64.deb' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_linux64bit.deb + cp '{{offline_repo_location}}/offline-installer-repo/src/dist/{{installer_version}}/linux/x64/{{environment_name}}_{{installer_version}}_amd64.deb' offline_artifacts/{{time}}/{{environment_name}}_{{installer_version}}_linux64bit.deb fi diff --git a/ansible/roles/openjdk/tasks/main.yml b/ansible/roles/openjdk/tasks/main.yml index e8b477298..150cfd9f4 100644 --- a/ansible/roles/openjdk/tasks/main.yml +++ b/ansible/roles/openjdk/tasks/main.yml @@ -1,15 +1,9 @@ - name: installing repo for open jdk in Ubuntu apt_repository: repo='ppa:openjdk-r/ppa' -- name: ensure oracle jdk 8 is absent - apt: name='oracle-java8-installer' state=absent - - name: ensure oracle jdk 8 set default is absent apt: name='oracle-java8-set-default' state=absent -- name: ensure oracle jdk 8 set default is absent - apt: name='openjdk-8-jre-headless' state=absent - - name: ensure openjdk 8 is present - apt: name='openjdk-8-jre-headless={{ java_version }}' state=present + apt: name='openjdk-8-jre-headless' state=present diff --git a/ansible/roles/postgres-azure-managed-service/tasks/main.yml b/ansible/roles/postgres-azure-managed-service/tasks/main.yml index d12786060..cc5ede8e0 100644 --- a/ansible/roles/postgres-azure-managed-service/tasks/main.yml +++ b/ansible/roles/postgres-azure-managed-service/tasks/main.yml @@ -46,14 +46,14 @@ command: az storage container create --name {{ postgresql_backup_azure_container_name }} ignore_errors: true environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" - name: Upload to azure blob storage command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }}.zip --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }} environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" async: 3600 poll: 10 diff --git a/ansible/roles/postgresql-backup/defaults/main.yml b/ansible/roles/postgresql-backup/defaults/main.yml index 1c2b49a0a..d64be512d 100644 --- a/ansible/roles/postgresql-backup/defaults/main.yml +++ b/ansible/roles/postgresql-backup/defaults/main.yml @@ -3,4 +3,3 @@ postgresql_user: postgres postgresql_backup_azure_container_name: postgresql-backup # Set these vars per environment as show in example below -# postgresql_backup_azure_storage_account_name: ntpbackupsstaging diff --git a/ansible/roles/postgresql-backup/tasks/main.yml b/ansible/roles/postgresql-backup/tasks/main.yml index 9b11fd451..c71f15510 100644 --- a/ansible/roles/postgresql-backup/tasks/main.yml +++ b/ansible/roles/postgresql-backup/tasks/main.yml @@ -17,16 +17,16 @@ command: az storage container create --name {{ postgresql_backup_azure_container_name }} ignore_errors: true environment: - AZURE_STORAGE_ACCOUNT: "{{ postgresql_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ postgresql_backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" - name: Upload to azure blob storage command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }} --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }} environment: - AZURE_STORAGE_ACCOUNT: "{{ postgresql_backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ postgresql_backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" async: 3600 poll: 10 - name: clean up backup dir after upload - file: path="{{ postgresql_backup_dir }}" state=absent \ No newline at end of file + file: path="{{ postgresql_backup_dir }}" state=absent diff --git a/ansible/roles/prometheus-backup/tasks/main.yml b/ansible/roles/prometheus-backup/tasks/main.yml index 9fda4a932..712dd6faf 100644 --- a/ansible/roles/prometheus-backup/tasks/main.yml +++ b/ansible/roles/prometheus-backup/tasks/main.yml @@ -29,14 +29,14 @@ command: az storage container create --name {{ prometheus_backup_azure_container_name }} ignore_errors: true environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" - name: Upload to azure blob storage command: az storage blob upload --name {{ prometheus_backup_gzip_file_name }} --file {{ prometheus_backup_gzip_file_path }} --container-name {{ prometheus_backup_azure_container_name }} environment: - AZURE_STORAGE_ACCOUNT: "{{ backup_azure_storage_account_name }}" - AZURE_STORAGE_KEY: "{{ backup_azure_storage_access_key }}" + AZURE_STORAGE_ACCOUNT: "{{ sunbird_management_storage_account_name }}" + AZURE_STORAGE_KEY: "{{ sunbird_management_storage_account_key }}" async: 3600 poll: 10 diff --git a/ansible/roles/redis-backup/defaults/main.yml b/ansible/roles/redis-backup/defaults/main.yml new file mode 100644 index 000000000..e00b84ce4 --- /dev/null +++ b/ansible/roles/redis-backup/defaults/main.yml @@ -0,0 +1,3 @@ +redis_backup_dir: /tmp/redis-backup +nodebb_redis_backup_azure_container_name: nodebb-redis-backup +learner_user: learning diff --git a/ansible/roles/mongo-backup/meta/main.yml b/ansible/roles/redis-backup/meta/main.yml similarity index 50% rename from ansible/roles/mongo-backup/meta/main.yml rename to ansible/roles/redis-backup/meta/main.yml index 23b18a800..a124d4f7c 100644 --- a/ansible/roles/mongo-backup/meta/main.yml +++ b/ansible/roles/redis-backup/meta/main.yml @@ -1,2 +1,2 @@ dependencies: - - azure-cli \ No newline at end of file + - azure-cli diff --git a/ansible/roles/redis-backup/tasks/main.yml b/ansible/roles/redis-backup/tasks/main.yml new file mode 100644 index 000000000..3519bb1ea --- /dev/null +++ b/ansible/roles/redis-backup/tasks/main.yml @@ -0,0 +1,37 @@ +- name: Create the directory + file: path={{ redis_backup_dir }} state=directory recurse=yes + + +- set_fact: + redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}.rdb" + +- set_fact: + redis_backup_file_path: "{{ redis_backup_dir }}/{{ redis_backup_file_name }}" + + +- name: copy dump.rdb file + copy: + src: /home/learning/redis-stable/dump.rdb + dest: "{{ redis_backup_dir }}/{{ redis_backup_file_name }}" + remote_src: yes + + +- name: upload to azure + include_role: + name: artifacts-upload-azure + vars: + artifact: "{{ redis_backup_file_name }}" + artifact_path: "{{ redis_backup_file_path }}" + artifacts_container: "{{ nodebb_redis_backup_azure_container_name }}" + +- name: clean up backup dir after upload + file: path={{ redis_backup_dir }} state=absent + +- name: Keyspace info + shell: "echo info keyspace | ./src/redis-cli" + register: restoreinfo + args: + chdir: "/home/learning/redis-stable/" + +- debug: var=restoreinfo.stdout_lines + diff --git a/ansible/roles/reloadIngress/tasks/kong.yml b/ansible/roles/reloadIngress/tasks/kong.yml index 21fc3b443..1d352aae4 100644 --- a/ansible/roles/reloadIngress/tasks/kong.yml +++ b/ansible/roles/reloadIngress/tasks/kong.yml @@ -1,6 +1,6 @@ --- - name: Getting the list of kong services host details - shell: "getent hosts $(docker service ps api-manager_kong | grep Runn | awk '{print $4}') | cut -d \" \" -f1| grep -v f" + shell: "getent ahostsv4 $(docker service ps api-manager_kong | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | uniq" register: kong_details - name: reload kong container configurations diff --git a/ansible/roles/reloadIngress/tasks/proxy.yml b/ansible/roles/reloadIngress/tasks/proxy.yml index c10bf18c3..09562cbc0 100644 --- a/ansible/roles/reloadIngress/tasks/proxy.yml +++ b/ansible/roles/reloadIngress/tasks/proxy.yml @@ -1,6 +1,6 @@ --- - name: Getting the list of proxy services host details - shell: "getent hosts $(docker service ps proxy_proxy | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | grep -v f " + shell: "getent ahostsv4 $(docker service ps proxy_proxy | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | uniq" register: proxy_details - name: reload proxy containers configurations diff --git a/ansible/roles/samza-jobs/tasks/main.yml b/ansible/roles/samza-jobs/tasks/main.yml index ad26aad48..8fab56622 100644 --- a/ansible/roles/samza-jobs/tasks/main.yml +++ b/ansible/roles/samza-jobs/tasks/main.yml @@ -63,7 +63,7 @@ - name: update environment specific details in new job configs replace: dest="{{ item[1].stdout }}" regexp="{{ item[0].key }}" replace="{{ item[0].value }}" with_nested: - - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__lms_host__", value: "{{__lms_host__}}"}, {key: "__lms_es_port__", value: "{{sunbird_es_port}}"}, {key: "__lms_es_host__", value: "{{sunbird_es_host}}"},{key: "__fcm_account_key__", value: "{{__fcm_account_key__}}"}] + - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__lms_host__", value: "{{__lms_host__}}"}, {key: "__lms_es_port__", value: "{{sunbird_es_port}}"}, {key: "__lms_es_host__", value: "{{sunbird_es_host}}"},{key: "__fcm_account_key__", value: "{{__fcm_account_key__}}"}, {key: "__sunbird_msg_91_auth__", value: "{{__sunbird_msg_91_auth__}}"}, {key: "__sunbird_notification_msg_default_sender__", value: "{{__sunbird_notification_msg_default_sender__}}"}, {key: "__sunbird_mail_server_from_email__", value: "{{__sunbird_mail_server_from_email__}}"}, {key: "__sunbird_mail_server_host__", value: "{{__sunbird_mail_server_host__}}"}, {key: "__sunbird_mail_server_password__", value: "{{__sunbird_mail_server_password__}}"}, {key: "__sunbird_mail_server_username__", value: "{{__sunbird_mail_server_username__}}"}, {key: "__sunbird_mail_server_port__", value: "{{__sunbird_mail_server_port__}}"}] - "{{ config_files | json_query('results[]') }}" - name: Start the jobs diff --git a/ansible/roles/stack-adminutil/defaults/main.yml b/ansible/roles/stack-adminutil/defaults/main.yml index a0f0d3126..79acae9c9 100644 --- a/ansible/roles/stack-adminutil/defaults/main.yml +++ b/ansible/roles/stack-adminutil/defaults/main.yml @@ -1,8 +1,9 @@ --- adminutil_replicas: 1 +adminutil_reserve_cpu: 0.2 +adminutil_limit_cpu: 0.5 adminutil_reservation_memory: 512M adminutil_limit_memory: 1024M - adminutil__port: 4000 adminutil__initial_heap_size: 256m adminutil__max_heap_size: 256m @@ -12,5 +13,4 @@ adminutil__health_id: apihealth adminutil__is_health_sensitive: false adminutil__metrics_id: metrics adminutil__is_metrics_sensitive: false - kong_url: "http://{{kong_host}}:8001" diff --git a/ansible/roles/stack-adminutil/templates/stack-adminutil.yml b/ansible/roles/stack-adminutil/templates/stack-adminutil.yml index efc505215..8b38da7e7 100644 --- a/ansible/roles/stack-adminutil/templates/stack-adminutil.yml +++ b/ansible/roles/stack-adminutil/templates/stack-adminutil.yml @@ -21,8 +21,10 @@ services: replicas: {{ adminutil_replicas }} resources: reservations: + cpus: "{{ adminutil_reserve_cpu }}" memory: "{{ adminutil_reservation_memory }}" limits: + cpus: "{{ adminutil_limit_cpu }}" memory: "{{ adminutil_limit_memory }}" update_config: parallelism: 1 diff --git a/ansible/roles/stack-api-manager/defaults/main.yml b/ansible/roles/stack-api-manager/defaults/main.yml index 1cd336a2d..535c5f063 100644 --- a/ansible/roles/stack-api-manager/defaults/main.yml +++ b/ansible/roles/stack-api-manager/defaults/main.yml @@ -4,8 +4,12 @@ kong_ssl: true kong_replicas: 1 kong_reservation_memory: 64M kong_limit_memory: 256M +kong_reserve_cpu: 0.3 +kong_limit_cpu: 1 echo_service_replicas: 1 echo_service_reservation_memory: 8M echo_service_limit_memory: 16M +echo_service_reserve_cpu: 0.1 +echo_service_limit_cpu: 0.2 kong_version: "{{kong_version}}" diff --git a/ansible/roles/stack-api-manager/templates/api-manager.env b/ansible/roles/stack-api-manager/templates/api-manager.env index 4d0ebf922..86ffb9a9b 100644 --- a/ansible/roles/stack-api-manager/templates/api-manager.env +++ b/ansible/roles/stack-api-manager/templates/api-manager.env @@ -5,3 +5,9 @@ KONG_PG_SSL={{ kong_ssl }} KONG_PG_USER={{ kong_postgres_user }} KONG_PG_PASSWORD={{ kong_postgres_password }} KONG_PG_DATABASE={{ kong_postgres_database }} +KONG_ADMIN_LISTEN=0.0.0.0:8001 +KONG_MEM_CACHE_SIZE={{ kong_mem_cache_size | default('128m') }} +KONG_TRUSTED_IPS=0.0.0.0/0,::/0 +KONG_UPSTREAM_KEEPALIVE={{ kong_upstream_keepalive | default('200') }} +KONG_NGINX_WORKER_PROCESSES={{ kong_nginx_worker_processes | default('auto')}} +KONG_RATELIMIT_CACHE_SIZE={{kong_ratelimit_cache_size | default('12m') }} diff --git a/ansible/roles/stack-api-manager/templates/stack-api-manager.yml b/ansible/roles/stack-api-manager/templates/stack-api-manager.yml index edab2c061..301c7f500 100644 --- a/ansible/roles/stack-api-manager/templates/stack-api-manager.yml +++ b/ansible/roles/stack-api-manager/templates/stack-api-manager.yml @@ -14,9 +14,11 @@ services: replicas: {{ kong_replicas }} resources: reservations: + cpus: "{{ kong_reserve_cpu }}" memory: "{{ kong_reservation_memory }}" limits: memory: "{{ kong_limit_memory }}" + cpus: "{{ kong_limit_cpu }}" update_config: parallelism: 1 delay: 30s @@ -32,8 +34,10 @@ services: resources: reservations: memory: "{{ echo_service_reservation_memory }}" + cpus: "{{ echo_service_reserve_cpu }}" limits: memory: "{{ echo_service_limit_memory }}" + cpus: "{{ echo_service_limit_cpu }}" update_config: parallelism: 1 delay: 5s diff --git a/ansible/roles/stack-badger/defaults/main.yml b/ansible/roles/stack-badger/defaults/main.yml index 1e5632417..5f31d0e32 100644 --- a/ansible/roles/stack-badger/defaults/main.yml +++ b/ansible/roles/stack-badger/defaults/main.yml @@ -1 +1,6 @@ badger_admin_user: admin +badger_replicas: 1 +badger_reservation_memory: 300MB +badger_limit_memory: 500MB +badger_reserve_cpu: 0.1 +badger_limit_cpu: 0.5 diff --git a/ansible/roles/stack-badger/tasks/.user.yml.swp b/ansible/roles/stack-badger/tasks/.user.yml.swp deleted file mode 100644 index a59000991..000000000 Binary files a/ansible/roles/stack-badger/tasks/.user.yml.swp and /dev/null differ diff --git a/ansible/roles/stack-badger/tasks/main.yml b/ansible/roles/stack-badger/tasks/main.yml index 805018736..6ded090df 100644 --- a/ansible/roles/stack-badger/tasks/main.yml +++ b/ansible/roles/stack-badger/tasks/main.yml @@ -28,7 +28,7 @@ - name: Deploy badger service become: yes - shell: "docker service create --with-registry-auth --replicas {{ badger_replicas }} -p 8004:8004 --name badger-service --hostname badger-service --reserve-memory {{ badger_reservation_memory }} --limit-memory {{ badger_limit_memory }} --network application_default --config source=settings_local.py,target=/badger/code/apps/mainsite/settings_local.py,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ badger_replicas }} -p 8004:8004 --name badger-service --hostname badger-service --reserve-cpu {{ badger_reserve_cpu }} --limit-cpu {{ badger_limit_cpu }} --reserve-memory {{ badger_reservation_memory }} --limit-memory {{ badger_limit_memory }} --network application_default --config source=settings_local.py,target=/badger/code/apps/mainsite/settings_local.py,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" args: chdir: /home/deployer/stack diff --git a/ansible/roles/stack-badger/templates/settings_local.py b/ansible/roles/stack-badger/templates/settings_local.py index d57fcaf93..8200772a3 100644 --- a/ansible/roles/stack-badger/templates/settings_local.py +++ b/ansible/roles/stack-badger/templates/settings_local.py @@ -105,8 +105,8 @@ ##AZURE CONFIGURATION### DEFAULT_FILE_STORAGE = "{{badger_file_storage}}" -AZURE_ACCOUNT_NAME = "{{sunbird_account_name}}" -AZURE_ACCOUNT_KEY = "{{sunbird_account_key}}" +AZURE_ACCOUNT_NAME = "{{sunbird_public_storage_account_name}}" +AZURE_ACCOUNT_KEY = "{{sunbird_public_storage_account_key}}" MEDIA_URL = "{{badger_url}}" AZURE_CONTAINER = "{{badger_container}}" diff --git a/ansible/roles/stack-kibana/defaults/main.yml b/ansible/roles/stack-kibana/defaults/main.yml index f2710e410..e4274d420 100644 --- a/ansible/roles/stack-kibana/defaults/main.yml +++ b/ansible/roles/stack-kibana/defaults/main.yml @@ -2,4 +2,3 @@ container_cpu_usage_percentage_theshold: 90 container_memory_usage_percentage_theshold: 90 logger_kibana_reservation_memory: 750M logger_kibana_limit_memory: 750M -kibana_image: kibana:4.6 diff --git a/ansible/roles/stack-kibana/tasks/main.yml b/ansible/roles/stack-kibana/tasks/main.yml index bb4afdd59..747bdc498 100644 --- a/ansible/roles/stack-kibana/tasks/main.yml +++ b/ansible/roles/stack-kibana/tasks/main.yml @@ -19,9 +19,8 @@ - name: Save kibana config template: src=kibana.yml dest=/home/deployer/config/kibana.yml mode=0644 -- name: Save kibana config - template: src=kibana-623.yml dest=/home/deployer/config/kibana-623.yml mode=0644 - when: "'log-es-2' is defined" +- name: Remove old kibana config + file: dest=/home/deployer/config/kibana-623.yml state=absent - name: Remove stack shell: "docker stack rm kibana" @@ -39,10 +38,6 @@ ignore_errors: yes when: "'log-es-2' is defined" -- name: Save kibana config as docker config - shell: "docker config create kibana-623.yml /home/deployer/config/kibana-623.yml" - when: "'log-es-2' is defined" - - name: Deploy stack shell: "docker stack deploy -c stack-kibana.yml kibana" args: diff --git a/ansible/roles/stack-kibana/templates/kibana-623.yml b/ansible/roles/stack-kibana/templates/kibana-623.yml deleted file mode 100644 index 5aaba467b..000000000 --- a/ansible/roles/stack-kibana/templates/kibana-623.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Kibana is served by a back end server. This controls which port to use. -server.port: 5601 - -server.basePath: "/dashboard" - -# The host to bind the server to. -server.host: "0.0.0.0" - -# The Elasticsearch instance to use for all your queries. -# This would use local elasticsearch as recommended in -# https://www.elastic.co/guide/en/kibana/current/production.html#load-balancing -elasticsearch.url: "http://{{logger_es6_host}}:{{logger_es_port}}" - -# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, -# then the host you use to connect to *this* Kibana instance will be sent. -elasticsearch.preserveHost: true - -# Kibana uses an index in Elasticsearch to store saved searches, visualizations -# and dashboards. It will create a new index if it doesn't already exist. -kibana.index: ".kibana" - -# If your Elasticsearch is protected with basic auth, this is the user credentials -# used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana -# users will still need to authenticate with Elasticsearch (which is proxied thorugh -# the Kibana server) -# kibana_elasticsearch_username: user -# kibana_elasticsearch_password: pass - - -# The default application to load. -kibana.defaultAppId: "dashboard" - -# Time in milliseconds to wait for responses from the back end or elasticsearch. -# This must be > 0 -elasticsearch.requestTimeout: 300000 - -# Time in milliseconds for Elasticsearch to wait for responses from shards. -# Set to 0 to disable. -elasticsearch.shardTimeout: 0 - -# Set to false to have a complete disregard for the validity of the SSL -# certificate. -elasticsearch.ssl.verificationMode: full - -# If you need to provide a CA certificate for your Elasticsarech instance, put -# the path of the pem file here. -# ca: /path/to/your/CA.pem - -# SSL for outgoing requests from the Kibana Server (PEM formatted) -# ssl_key_file: /path/to/your/server.key -# ssl_cert_file: /path/to/your/server.crt - -# Set the path to where you would like the process id file to be created. -# pid.file: /var/run/kibana.pid - diff --git a/ansible/roles/stack-kibana/templates/kibana.yml b/ansible/roles/stack-kibana/templates/kibana.yml index 2bf6b0f47..d09e336fb 100644 --- a/ansible/roles/stack-kibana/templates/kibana.yml +++ b/ansible/roles/stack-kibana/templates/kibana.yml @@ -1,26 +1,26 @@ # Kibana is served by a back end server. This controls which port to use. -{% if groups['log-es-2'] is defined %} -port: 5602 -server.basePath: "/dashboard-old" -{% else %} -port: 5601 +server.port: 5601 + server.basePath: "/dashboard" -{% endif %} +server.rewriteBasePath: true +xpack.ilm.enabled: false +xpack.security.enabled: false + # The host to bind the server to. -host: "0.0.0.0" +server.host: "0.0.0.0" # The Elasticsearch instance to use for all your queries. # This would use local elasticsearch as recommended in # https://www.elastic.co/guide/en/kibana/current/production.html#load-balancing -elasticsearch_url: "http://{{logger_es_host}}:{{logger_es_port}}" +elasticsearch.url: "http://{{logger_es6_host}}:{{logger_es_port}}" # preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, # then the host you use to connect to *this* Kibana instance will be sent. -elasticsearch_preserve_host: true +elasticsearch.preserveHost: true # Kibana uses an index in Elasticsearch to store saved searches, visualizations # and dashboards. It will create a new index if it doesn't already exist. -kibana_index: ".kibana" +kibana.index: ".kibana" # If your Elasticsearch is protected with basic auth, this is the user credentials # used by the Kibana server to perform maintence on the kibana_index at statup. Your Kibana @@ -31,19 +31,19 @@ kibana_index: ".kibana" # The default application to load. -default_app_id: "dashboard" +kibana.defaultAppId: "discover" # Time in milliseconds to wait for responses from the back end or elasticsearch. # This must be > 0 -request_timeout: 300000 +elasticsearch.requestTimeout: 300000 # Time in milliseconds for Elasticsearch to wait for responses from shards. # Set to 0 to disable. -shard_timeout: 0 +elasticsearch.shardTimeout: 0 # Set to false to have a complete disregard for the validity of the SSL # certificate. -verify_ssl: true +elasticsearch.ssl.verificationMode: full # If you need to provide a CA certificate for your Elasticsarech instance, put # the path of the pem file here. @@ -54,17 +54,5 @@ verify_ssl: true # ssl_cert_file: /path/to/your/server.crt # Set the path to where you would like the process id file to be created. -# pid_file: /var/run/kibana.pid +# pid.file: /var/run/kibana.pid -# Plugins that are included in the build, and no longer found in the plugins/ folder -bundled_plugin_ids: - - plugins/dashboard/index - - plugins/discover/index - - plugins/doc/index - - plugins/kibana/index - - plugins/markdown_vis/index - - plugins/metric_vis/index - - plugins/settings/index - - plugins/table_vis/index - - plugins/vis_types/index - - plugins/visualize/index diff --git a/ansible/roles/stack-kibana/templates/stack-kibana.yml b/ansible/roles/stack-kibana/templates/stack-kibana.yml index 7471d37d1..0c84def5d 100644 --- a/ansible/roles/stack-kibana/templates/stack-kibana.yml +++ b/ansible/roles/stack-kibana/templates/stack-kibana.yml @@ -1,54 +1,27 @@ version: '3.3' services: - kibana_old: - image: kibana:4.6 - deploy: - resources: - reservations: - memory: "750M" - limits: - memory: "750M" - environment: - - LOGSPOUT=ignore -{% if groups['log-es-2'] is defined %} - ports: - - '5602:5602' -{% else %} - ports: - - '5601:5601' -{% endif %} - configs: - - source: kibana.yml - target: /opt/kibana/config/kibana.yml - networks: - - application_default -{% if groups['log-es-2'] is defined %} kibana: - image: sunbird/kibana:6.2.3 + image: kibana:6.8.6 deploy: resources: reservations: memory: "750M" limits: - memory: "750M" + memory: "3072M" environment: - LOGSPOUT=ignore ports: - '5601:5601' configs: - - source: kibana-623.yml + - source: kibana.yml target: /opt/kibana/config/kibana.yml networks: - application_default -{% endif %} + configs: kibana.yml: external: true -{% if groups['log-es-2'] is defined %} - kibana-623.yml: - external: true -{% endif %} networks: application_default: external: true diff --git a/ansible/roles/stack-logger/defaults/main.yml b/ansible/roles/stack-logger/defaults/main.yml index f21a32040..d9bef3e8d 100644 --- a/ansible/roles/stack-logger/defaults/main.yml +++ b/ansible/roles/stack-logger/defaults/main.yml @@ -1,9 +1,11 @@ # Please change memory requirements if heap_size is changed logger_logstash_heap_size: 512M -logger_logstash_replicas: 2 logger_logstash_reservation_memory: 512M -logger_logstash_limit_memory: 750M +logger_logstash_limit_memory: 512M +logger_logstash_reservation_cpu: 0.2 +logger_logstash_limit_cpu: 0.5 + logger_logspout_reservation_memory: 75M -logger_logspout_limit_memory: 75M -logger_logstash_reservation_memory: 1024M -logger_logstash_limit_memory: 1024M +logger_logspout_limit_memory: 150M +logger_logspout_reservation_cpu: 0.1 +logger_logspout_limit_cpu: 0.2 diff --git a/ansible/roles/stack-logger/templates/stack-logger.yml b/ansible/roles/stack-logger/templates/stack-logger.yml index 29f1d77b8..1b3b51048 100644 --- a/ansible/roles/stack-logger/templates/stack-logger.yml +++ b/ansible/roles/stack-logger/templates/stack-logger.yml @@ -6,12 +6,14 @@ services: image: sunbird/logstash:6.2.3 command: /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/logstash.conf deploy: - replicas: {{ logger_logstash_replicas }} + mode: global resources: reservations: memory: "{{ logger_logstash_reservation_memory }}" + cpus: "{{ logger_logstash_reservation_cpu }}" limits: memory: "{{ logger_logstash_limit_memory }}" + cpus: "{{ logger_logstash_limit_cpu }}" environment: - LOGSPOUT=ignore - LS_HEAP_SIZE={{ logger_logstash_heap_size }} @@ -27,12 +29,14 @@ services: image: sunbird/logstash:2.4.1 command: logstash -f /conf/logstash.conf deploy: - replicas: {{ logger_logstash_replicas }} + mode: gloabl resources: reservations: memory: "{{ logger_logstash_reservation_memory }}" + cpus: "{{ logger_logspout_reservation_cpu }}" limits: memory: "{{ logger_logstash_limit_memory }}" + cpus: "{{ logger_logspout_limit_cpu }}" environment: - LOGSPOUT=ignore - LS_HEAP_SIZE={{ logger_logstash_heap_size }} @@ -54,8 +58,10 @@ services: resources: reservations: memory: "{{ logger_logspout_reservation_memory }}" + cpus: "{{ logger_logspout_reservation_cpu }}" limits: memory: "{{ logger_logspout_limit_memory }}" + cpus: "{{ logger_logspout_limit_cpu }}" environment: - SYSLOG_FORMAT=rfc3164 - INACTIVITY_TIMEOUT=1m diff --git a/ansible/roles/stack-monitor/templates/data_backup_azure_blob_exporter_config.yml b/ansible/roles/stack-monitor/templates/data_backup_azure_blob_exporter_config.yml index f58eb9371..99eb63701 100644 --- a/ansible/roles/stack-monitor/templates/data_backup_azure_blob_exporter_config.yml +++ b/ansible/roles/stack-monitor/templates/data_backup_azure_blob_exporter_config.yml @@ -1,5 +1,5 @@ exporter_port: 9358 log_level: info -azure_blob_storage_account_name: '{{ backup_storage_name }}' -azure_blob_storage_account_key: '{{ backup_storage_key }}' +sunbird_management_storage_account_name: "{{ sunbird_management_storage_account_name }}" +sunbird_management_storage_account_key: "{{ sunbird_management_storage_account_key }}" azure_blob_storage_containers: ['cassandra-backup', 'postgresql-backup'] diff --git a/ansible/roles/stack-monitor/templates/stack-monitor.yml b/ansible/roles/stack-monitor/templates/stack-monitor.yml index 281f83b3d..36532a6c4 100644 --- a/ansible/roles/stack-monitor/templates/stack-monitor.yml +++ b/ansible/roles/stack-monitor/templates/stack-monitor.yml @@ -93,7 +93,7 @@ services: limits: memory: "{{ cadvisor_limit_memory }}" - statsd_exporter: + statsd-service: image: quay.io/prometheus/statsd-exporter:v0.9.0 ports: - 9102:9102 diff --git a/ansible/roles/stack-oauth/defaults/main.yml b/ansible/roles/stack-oauth/defaults/main.yml index cccc522e0..f4a607436 100644 --- a/ansible/roles/stack-oauth/defaults/main.yml +++ b/ansible/roles/stack-oauth/defaults/main.yml @@ -7,4 +7,4 @@ logger_oauth_limit_memory: 32M kibana_oauth_authenticated_email_domains: [] kibana_oauth_authenticated_email_ids: [] -swarm_agent_for_proxy: "{{groups['swarm-worker'][0]}}" \ No newline at end of file +swarm_agent_for_proxy: "{{groups['swarm-worker'][0]}}" diff --git a/ansible/roles/stack-oauth/templates/stack-oauth.yml b/ansible/roles/stack-oauth/templates/stack-oauth.yml index da04e2702..518558499 100644 --- a/ansible/roles/stack-oauth/templates/stack-oauth.yml +++ b/ansible/roles/stack-oauth/templates/stack-oauth.yml @@ -2,7 +2,7 @@ version: '3.3' services: oauth: - image: rjshrjndrn/oauth2-proxy:2.2 + image: sunbird/oauth2_proxy:v1.0 ports: - "4111:4111" deploy: @@ -12,7 +12,7 @@ services: limits: memory: "{{ logger_oauth_limit_memory }}" command: | - -cookie-secure=false + -cookie-secure=true -upstream=http://{{swarm_agent_for_proxy}}/oauth2 -redirect-url={{kibana_oauth_redirect_url}} -http-address=0.0.0.0:4111 @@ -20,6 +20,12 @@ services: -email-domain={{email_domain}} {% endfor %} -authenticated-emails-file=/authenticated_email.txt + -cookie-secret={{cookie_secret}} + -cookie-expire=24h + -cookie-refresh=4h + -cookie-httponly=true + -silence-ping-logging=true + -skip-provider-button=true environment: - OAUTH2_PROXY_CLIENT_ID={{google_client_id}} - OAUTH2_PROXY_CLIENT_SECRET={{google_client_secret}} diff --git a/ansible/roles/stack-proxy-private/defaults/main.yml b/ansible/roles/stack-proxy-private/defaults/main.yml new file mode 100644 index 000000000..92107bf47 --- /dev/null +++ b/ansible/roles/stack-proxy-private/defaults/main.yml @@ -0,0 +1,15 @@ +--- +hub_org: sunbird +nginx_per_ip_connection_limit: 400 +private_proxy_replicas: 1 +private_proxy_reservation_memory: 64M +private_proxy_limit_memory: 128M +private_proxy_reserve_cpu: 0.1 +private_proxy_limit_cpu: 0.3 +nginx_per_ip_connection_limit: 400 +merge_proxy_server_name: +proxy_prometheus: false + +networks: + - {name: 'application_default', subnet: '11.11.1.0/24'} + diff --git a/ansible/roles/stack-proxy-private/tasks/main.yml b/ansible/roles/stack-proxy-private/tasks/main.yml new file mode 100644 index 000000000..37ac574a5 --- /dev/null +++ b/ansible/roles/stack-proxy-private/tasks/main.yml @@ -0,0 +1,44 @@ +--- +- name: Ensure stack directory exists + file: + path: /home/deployer/stack + state: directory + owner: root + group: root + +- name: Ensure config directory exists + file: + path: /home/deployer/config + state: directory + owner: root + group: root + +- name: Save stack file + template: src=stack-proxy.yml dest=/home/deployer/stack/proxy-private.yml mode=0644 + +- name: Remove stack + shell: "docker stack rm private" + ignore_errors: yes + +- name: Save proxy-default.conf + template: src=proxy-default.conf dest=/home/deployer/config/proxy-default-private.conf mode=0644 + +- name: Ensure network exists + shell: "docker network create --driver overlay {{item.name}} --subnet {{item.subnet}}" + with_items: + - "{{networks}}" + ignore_errors: true + +- name: Remove old proxy-default-private.conf docker config + shell: "docker config rm proxy-default-private.conf" + ignore_errors: yes + +- name: Save proxy-default-private.conf as docker config + shell: "docker config create proxy-default-private.conf /home/deployer/config/proxy-default-private.conf" + +- debug: msg="server name {{proxy_server_name}}" + +- name: Deploy stack + shell: "docker stack deploy -c proxy-private.yml private" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-proxy-private/templates/proxy-default.conf b/ansible/roles/stack-proxy-private/templates/proxy-default.conf new file mode 100644 index 000000000..60433f936 --- /dev/null +++ b/ansible/roles/stack-proxy-private/templates/proxy-default.conf @@ -0,0 +1,47 @@ +server { + listen 80; + listen [::]:80; + + resolver 127.0.0.11 valid=5s; + + location /learner/ { + rewrite ^/learner/(.*) /$1 break; + proxy_pass http://learner-service:9000; + } + location /api/ { + rewrite ^/api/(.*) /$1 break; + proxy_pass http://kong:8000; + } + location /admin-api/ { + rewrite ^/admin-api/(.*) /$1 break; + proxy_pass http://kong:8001; + } + location /content/ { + rewrite ^/content/(.*) /$1 break; + proxy_pass http://content-service:9000; + } + location /badger/ { + rewrite ^/badger/(.*) /$1 break; + proxy_pass http://badger-service:8004; + } + location /cert/ { + rewrite ^/cert/(.*) /$1 break; + proxy_pass http://cert-service:9000; + } + location /certreg/ { + rewrite ^/certreg/(.*) /$1 break; + proxy_pass http://cert-registry-service:9000; + } + location /print/ { + rewrite ^/print/(.*) /$1 break; + proxy_pass http://print-service:5000; + } + location /assessment/ { + rewrite ^/assessment/(.*) /$1 break; + proxy_pass http://assessment-service:9000; + } + location /notification/ { + rewrite ^/notification/(.*) /$1 break; + proxy_pass http://notification-service:9000; + } +} diff --git a/ansible/roles/stack-proxy-private/templates/stack-proxy.yml b/ansible/roles/stack-proxy-private/templates/stack-proxy.yml new file mode 100644 index 000000000..15df509a8 --- /dev/null +++ b/ansible/roles/stack-proxy-private/templates/stack-proxy.yml @@ -0,0 +1,32 @@ +version: '3.3' + +services: + proxy: + image: nginx + # This is a workaround for remote ip address is not visible nginx + ports: + - "31480:80" + deploy: + replicas: {{ private_proxy_replicas }} + resources: + reservations: + memory: "{{ private_proxy_reservation_memory }}" + cpus: "{{ private_proxy_reserve_cpu }}" + limits: + memory: "{{ private_proxy_limit_memory }}" + cpus: "{{ private_proxy_limit_cpu }}" + update_config: + parallelism: 1 + delay: 30s + configs: + - source: proxy-default-private.conf + target: /etc/nginx/conf.d/default.conf + networks: + - application_default +configs: + proxy-default-private.conf: + external: true + +networks: + application_default: + external: true diff --git a/ansible/roles/stack-proxy/defaults/main.yml b/ansible/roles/stack-proxy/defaults/main.yml index 8019d1971..579709e41 100644 --- a/ansible/roles/stack-proxy/defaults/main.yml +++ b/ansible/roles/stack-proxy/defaults/main.yml @@ -2,8 +2,10 @@ hub_org: sunbird nginx_per_ip_connection_limit: 400 proxy_replicas: 1 -proxy_reservation_memory: 32M -proxy_limit_memory: 64M +proxy_reservation_memory: 64M +proxy_limit_memory: 128M +proxy_reserve_cpu: 0.1 +proxy_limit_cpu: 0.3 nginx_per_ip_connection_limit: 400 merge_proxy_server_name: proxy_prometheus: false @@ -35,8 +37,8 @@ prometheus_route_prefix: prometheus prometheus_alertmanager_route_prefix: alertmanager ekstep_s3_env: "{{sunbird_env}}" registry_url: "{{proto}}://{{proxy_server_name}}/registry" -ep_es_host: +kibana_service: "{{swarm_dashboard}}:5601" upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" plugin_upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" -sunbird_offline_azure_storage_account_url: "{{ sunbird_offline_azure_storage_account }}.blob.core.windows.net/{{ offline_installer_container_name }}" +sunbird_offline_azure_storage_account_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ offline_installer_container_name }}" diff --git a/ansible/roles/stack-proxy/templates/nginx.conf b/ansible/roles/stack-proxy/templates/nginx.conf index 075189a14..cffade198 100644 --- a/ansible/roles/stack-proxy/templates/nginx.conf +++ b/ansible/roles/stack-proxy/templates/nginx.conf @@ -1,6 +1,15 @@ user nginx; worker_processes 1; +{# +Can add custom modules like +eg: +nginx_modules: | + load_module modules/ngx_http_geoip2_module.so; + load_module modules/ngx_stream_geoip2_module.so; +#} +{{ nginx_modules | d('') }} + error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; @@ -12,7 +21,21 @@ events { http { include /etc/nginx/mime.types; default_type application/octet-stream; - + {# + This is to define custom nginx_http_configs + for example + nginx_http_config: | + geoip2 /usr/local/share/GeoLite2-Country.mmdb { + $geoip2_data_country_iso_code country iso_code; + } + map $geoip2_data_country_iso_code $allowed_country { + default no; + IN no; + } + #} +{% if nginx_http_config is defined and nginx_http_config %} + {{ nginx_http_config | indent( width=5, indentfirst=True) }} +{% endif %} lua_load_resty_core off; log_format main '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent ' @@ -66,6 +89,14 @@ http { keepalive 1000; } + # local caching for images and files + proxy_cache_path /tmp/proxy_cache levels=1:2 keys_zone=tmp_cache:5m max_size=10m inactive=60m use_temp_path=off; + + proxy_cache_path /tmp/api_cache levels=1:2 keys_zone=proxy_cache:5m max_size=300m inactive=60m use_temp_path=off; + # cache framework + proxy_cache_path /tmp/framework_cache levels=1:2 keys_zone=framework_cache:5m max_size=700m inactive=60m use_temp_path=off; + + include /etc/nginx/conf.d/*.conf; server { diff --git a/ansible/roles/stack-proxy/templates/proxy-default.conf b/ansible/roles/stack-proxy/templates/proxy-default.conf index 6ac6da537..f98ba2aae 100644 --- a/ansible/roles/stack-proxy/templates/proxy-default.conf +++ b/ansible/roles/stack-proxy/templates/proxy-default.conf @@ -3,9 +3,19 @@ server { listen 80; listen [::]:80; server_name {{ proxy_server_name }}; + {# + custom nginx server config section + eg: + nginx_server_config: | + if ($allowed_country = no) { + return 444; + } + #} +{% if nginx_server_config is defined and nginx_server_config %} + {{ nginx_server_config | indent( width=2, indentfirst=True) }} +{% endif %} # Limitting open connection per ip limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; - return 301 https://{{ proxy_server_name }}$request_uri; } {% endif %} @@ -17,14 +27,27 @@ server { listen 443 ssl; ssl_certificate /run/secrets/site.crt; ssl_certificate_key /run/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; {% endif %} server_name {{ proxy_server_name }}; + {# + custom nginx server config section + eg: + nginx_server_config: | + if ($allowed_country = no) { + return 444; + } + #} +{% if nginx_server_config is defined and nginx_server_config %} + {{ nginx_server_config | indent( width=2, indentfirst=True) }} +{% endif %} + # Limitting open connection per ip limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-SSL on; proxy_set_header X-Forwarded-Proto $scheme; @@ -92,15 +115,44 @@ server { proxy_set_header X-Forwarded-Proto $scheme; # root /usr/share/nginx/www; } + + location /api/framework { + # Enabling caching + proxy_cache_key $proxy_host$request_uri; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + proxy_cache_valid 200 3600s; + + rewrite ^/api/(.*) /$1 break; + client_max_body_size 60M; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://kong; + } + location /api/ { if ($request_method = OPTIONS ) { add_header Access-Control-Allow-Origin "*" ; add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; - add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Content-Encoding"; add_header Content-Length 0; add_header Content-Type text/plain; return 200; } + if ( $arg_eHVyhwSdt ) { + set $custom_header "Bearer $arg_eHVyhwSdt"; + } + if ( $http_authorization ) { + set $custom_header "$http_authorization"; + } + proxy_set_header Authorization $custom_header; client_max_body_size 60M; rewrite ^/api/(.*) /$1 break; proxy_set_header Connection ""; @@ -157,60 +209,21 @@ server { # if you enabled --cookie-refresh, this is needed for it to work with auth_request auth_request_set $auth_cookie $upstream_http_set_cookie; add_header Set-Cookie $auth_cookie; - rewrite ^/dashboard/(.*) /$1 break; - proxy_pass $target; - - } - - location /dashboard-old/ { - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - # Setting target url - auth_request_set $target http://{{swarm_dashboard}}:5602; - # pass information via X-User and X-Email headers to backend, - # requires running with --set-xauthrequest flag - auth_request_set $user $upstream_http_x_auth_request_user; - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-User $user; - proxy_set_header X-Email $email; - - # if you enabled --cookie-refresh, this is needed for it to work with auth_request - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - rewrite ^/dashboard-old/(.*) /$1 break; proxy_pass $target; - } - location /pipeline-dashboard/ { - auth_request /oauth2/auth; - error_page 401 = /oauth2/sign_in; - - # Setting Target url - auth_request_set $target http://{{ ep_es_host}}:5601; - - # pass information via X-User and X-Email headers to backend, - # requires running with --set-xauthrequest flag - auth_request_set $user $upstream_http_x_auth_request_user; - auth_request_set $email $upstream_http_x_auth_request_email; - proxy_set_header X-User $user; - proxy_set_header X-Email $email; - - # if you enabled --cookie-refresh, this is needed for it to work with auth_request - auth_request_set $auth_cookie $upstream_http_set_cookie; - add_header Set-Cookie $auth_cookie; - - rewrite ^/pipeline-dashboard/(.*) /$1 break; - proxy_pass $target; - +{% if private_ingressgateway_ip is defined %} + location /grafana/ { + rewrite ^/grafana/(.*) /$1 break; + proxy_pass http://{{private_ingressgateway_ip}}:30300; } - +{% else %} location /grafana/ { set $target http://{{swarm_dashboard}}:3001; rewrite ^/grafana/(.*) /$1 break; proxy_pass $target; } +{% endif %} {% if proxy_prometheus==true %} location /{{prometheus_alertmanager_route_prefix}}/ { @@ -492,12 +505,46 @@ location ~* ^/desktop/(.*) { proxy_send_timeout 60; proxy_read_timeout 70; } + + location ~ /resourcebundles/v1/read|/learner/data/v1/(role/read|system/settings/get)|/v1/tenant/info { + proxy_cache_key $proxy_host$request_uri; + proxy_cache proxy_cache; + add_header X-Proxy-Cache $upstream_cache_status; + proxy_cache_valid 200 3600s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://player; + } + + location /api/channel/v1/read { + proxy_cache_key $proxy_host$request_uri; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + proxy_cache_valid 200 3600s; + + rewrite ^/api/channel/v1/read/(.*) /channel/v1/read/$1 break; + client_max_body_size 60M; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://kong; + } location /oauth2callback { return 200 'OK'; add_header Content-Type text/plain; } - + {# Including custom configuration #} {{ proxy_custom_config }} client_max_body_size 60M; diff --git a/ansible/roles/stack-proxy/templates/stack-proxy.yml b/ansible/roles/stack-proxy/templates/stack-proxy.yml index bc449d111..b13154fd0 100644 --- a/ansible/roles/stack-proxy/templates/stack-proxy.yml +++ b/ansible/roles/stack-proxy/templates/stack-proxy.yml @@ -7,19 +7,21 @@ services: ports: - mode: host target: 80 - published: 80 + published: 31380 protocol: TCP - mode: host target: 443 - published: 443 + published: 31390 protocol: TCP deploy: mode: global resources: reservations: memory: "{{ proxy_reservation_memory }}" + cpus: "{{ proxy_reserve_cpu }}" limits: memory: "{{ proxy_limit_memory }}" + cpus: "{{ proxy_limit_cpu }}" update_config: parallelism: 1 delay: 30s diff --git a/ansible/roles/stack-sunbird/defaults/main.yml b/ansible/roles/stack-sunbird/defaults/main.yml index 56af749b5..cd27fce92 100644 --- a/ansible/roles/stack-sunbird/defaults/main.yml +++ b/ansible/roles/stack-sunbird/defaults/main.yml @@ -1,4 +1,5 @@ --- +###################### Swarm variables ##################### sunbird_encryption_key: sunbird_encryption_mode: sunbird_sso_publickey: @@ -16,9 +17,11 @@ actor_reservation_memory: 768M actor_limit_memory: 1024M learner_replicas: 1 -learner_reservation_memory: 256M -learner_limit_memory: 512M -learner_reservation_cpu: 0 +learner_reservation_memory: 512M +learner_limit_memory: 1024M +learner_reservation_cpu: 0.2 +learner_limit_cpu: 1 +learner_java_mem_limit: '-Xmx600m' #Config Service Vars config_replicas: 1 @@ -31,6 +34,22 @@ config_reservation_cpu: 0 notification_replicas: 1 notification_limit_memory: 500MB notification_limit_cpu: 1 +notification_reserve_memory: 300MB +notification_reserve_cpu: 0.1 + +#Report service vars +report_replicas: 1 +report_reservation_memory: 300M +report_reservation_cpu: 0.1 +report_limit_memory: 500M +report_limit_cpu: 1 +sunbird_report_summary_table_name: report_summary + +player_replicas: 1 +player_reservation_memory: 128M +player_limit_memory: 750M +player_reserve_cpu: 0.1 +player_limit_cpu: 1 # This variable is not to access learner service but to call the api # learner-service:9000/org/v1/search @@ -39,16 +58,12 @@ sunbird_learner_service_base_url: http://kong:8000/ telemetry_replicas: 1 telemetry_reservation_memory: 300M telemetry_limit_memory: 300M +telemetry_reservation_cpu: 0.1 +telemetry_limit_cpu: 1 sunbird_telemetry_kafka_servers: "{{groups['kafka']|join(':9092,')}}:9092" sunbird_data_pipeline_kafka_servers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" -player_replicas: 1 -player_reservation_memory: 64M -player_limit_memory: 256M sunbird_build_number: -content_replicas: 1 -content_reservation_memory: 64M -content_limit_memory: 256M keycloak_auth_server_url: "{{proto}}://{{proxy_server_name}}/auth" keycloak_realm: sunbird sunbird_web_url: "{{proto}}://{{proxy_server_name}}" @@ -56,8 +71,8 @@ sunbird_cassandra_port: 9042 sunbird_portal_title_name: 'portal' sunbird_keycloak_client_id: 'portal' vault_sunbird_api_auth_token: -sunbird_azure_storage_account: -sunbird_azure_storage_key: +sunbird_public_storage_account_name: +sunbird_public_storage_account_key: sunbird_container_name: sunbird_portal_cdn_url: sunbird_dataservice_url: @@ -69,7 +84,7 @@ sunbird_telemetry_dispatchers: kafka content_service_whitelisted_channels: content_service_blacklisted_channels: sunbird_env_logo_url: -desktop_app_storage_url: "https://{{sunbird_offline_azure_storage_account}}.blob.core.windows.net/{{offline_installer_container_name}}" +desktop_app_storage_url: "https://{{sunbird_public_storage_account_name}}.blob.core.windows.net/{{offline_installer_container_name}}" telemetry_logstash_heap_size: 512m telemetry_logstash_replicas: 1 @@ -77,8 +92,6 @@ telemetry_logstash_reservation_memory: 1g telemetry_logstash_limit_memory: 2g sunbird_telemetry_api: sunbird_enable_signup: -content_limit_cpu: 1 -content_reservation_cpu: 0 user_org_replicas: 1 user_org_reservation_memory: 750MB @@ -90,23 +103,24 @@ cert_replicas: 1 cert_reservation_memory: 500MB cert_limit_memory: 550MB cert_limit_cpu: 1 -cert_reservation_cpu: 0 +cert_reservation_cpu: 0.1 cert_registry_replicas: 1 -cert_registry_limit_memory: 500MB -cert_registry_limit_cpu: 1 +cert_registry_limit_memory: 512MB +cert_registry_limit_cpu: 0.5 +cert_registry_reserve_cpu: 256MB +cert_registry_reserve_memory: 0.1 # Encryption service vars enc_replicas: 1 -enc_reservation_memory: 750MB -enc_limit_memory: 800MB -enc_limit_cpu: 1 -enc_reservation_cpu: 0 +enc_reservation_memory: 300MB +enc_limit_memory: 500MB +enc_limit_cpu: 0.5 +enc_reservation_cpu: 0.1 postgres_port: 5432 enc_dialect: postgres enc_entry_passwod: password - telemetry_service_threads: telemetry_local_storage_enabled: telemetry_local_storage_type: @@ -121,6 +135,10 @@ sunbird_time_zone: "Asia/Kolkata" sunbird_content_service_log_level: info sunbird_response_cache_ttl: 180 +#adminutil +adminutil_base_url: http://adminutil:4000/ +adminutil_sign_endpoint: v1/sign/payload + #learner sunbird_time_zone: "Asia/Kolkata" @@ -129,7 +147,634 @@ max_kafka_message_size: "5242880" sunbird_gzip_enable: true sunbird_analytics_blob_account_name: sunbird_analytics_blob_account_key: -sunbird_portal_player_cdn_enabled: +sunbird_portal_player_cdn_enabled: sunbird_portal_preview_cdn_url: cdn_file_path: sunbird_portal_cdn_blob_url: +itext_license_enabled: false + +# Knowledge MW Service Config +knowledge_mw_service_replicas: 1 +knowledge_mw_service_reservation_memory: 200M +knowledge_mw_service_limit_memory: 1000M +knowledge_mw_service_reservation_cpu: 0.1 +knowledge_mw_service_limit_cpu: 1 + +# Content Service Config +content_service_replicas: 1 +content_service_reservation_memory: 600M +content_service_limit_memory: 1000M +content_service_reservation_cpu: 0.1 +content_service_limit_cpu: 1 +content_java_mem_limit: '-Xmx600m' + +# Assessment Service Config +assessment_service_replicas: 1 +assessment_service_reservation_memory: 256M +assessment_service_limit_memory: 512M +assessment_service_reservation_cpu: 0.1 +assessment_service_limit_cpu: 1 + +# LMS Service Config +lms_service_replicas: 1 +lms_service_reservation_memory: 1000M +lms_service_limit_memory: 1000M +lms_service_reservation_cpu: 0.1 +lms_service_limit_cpu: 1 +lms_java_mem_limit: '-Xmx600m' + +# Print Service Config +print_service_replicas: 1 +print_service_reservation_memory: 128M +print_service_limit_memory: 512M +print_service_reservation_cpu: 0.1 +print_service_limit_cpu: 0.5 + +# Search Service Config +search_service_replicas: 1 +search_service_reservation_memory: 1000M +search_service_limit_memory: 1000M +search_service_reservation_cpu: 0 +search_service_limit_cpu: 1 + +# Taxonomy Service Config +taxonomy_service_replicas: 1 +taxonomy_service_reservation_memory: 256M +taxonomy_service_limit_memory: 512M +taxonomy_service_reservation_cpu: 0 +taxonomy_service_limit_cpu: 1 + +############################ kube vars ####################### +hub_org: sunbird +nginx_per_ip_connection_limit: 400 +proxy_replicas: 1 +proxy_reservation_memory: 32M +proxy_limit_memory: 64M +nginx_per_ip_connection_limit: 400 +merge_proxy_server_name: +proxy_prometheus: false + +networks: + - {name: 'application_default', subnet: '11.11.1.0/24'} + +proxy_secrets: + - name: prom_admin_creds + value: "{{proxy_prometheus_admin_creds}}" + - name: site.key + value: "{{proxy_site_key}}" + - name: site.crt + value: "{{proxy_site_crt}}" + +merge_proxy_secrets: + - name: sitemerge.key + value: "{{proxymerge_site_key}}" + - name: sitemerge.crt + value: "{{proxymerge_site_crt}}" + +proxy_custom_config: + +# These values should be overridden in inventory +proxy_site_key: +proxy_site_crt: +proxy_prometheus_admin_creds: +prometheus_route_prefix: prometheus +prometheus_alertmanager_route_prefix: alertmanager +ekstep_s3_env: "{{sunbird_env}}" +registry_url: "{{proto}}://{{proxy_server_name}}/registry" + +upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +plugin_upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +sunbird_offline_azure_storage_account_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ offline_installer_container_name }}" +# Override this dictionary in your common.yaml +proxy: + # repository: 'proxy' + # image_tag: 2.4.0 + +sunbird_portal_player_cdn_enabled: false +sunbird_cert_qr_container_name: "certqr" + +service_env: + groups: ../../../../ansible/roles/stack-sunbird/templates/sunbird_groups-service.env + learner: + - ../../../../ansible/roles/stack-sunbird/templates/sunbird_learner-service.env + - ../../../../ansible/roles/stack-sunbird/templates/learner-service_logback.xml + lms: + - ../../../../ansible/roles/stack-sunbird/templates/sunbird_lms-service.env + - ../../../../ansible/roles/stack-sunbird/templates/lms-service_logback.xml + knowledgemw: ../../../../ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env + apimanager: ../../../../ansible/roles/stack-api-manager/templates/api-manager.env + cert: ../../../../ansible/roles/stack-sunbird/templates/sunbird_cert-service.env + certregistry: ../../../../ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env + content: + - ../../../../ansible/roles/stack-sunbird/templates/content-service_application.conf + - ../../../../ansible/roles/stack-sunbird/templates/content-service_logback.xml + assessment: + - ../../../../ansible/roles/stack-sunbird/templates/assessment-service_application.conf + - ../../../../ansible/roles/stack-sunbird/templates/assessment-service_logback.xml + enc: ../../../../ansible/roles/stack-sunbird/templates/sunbird_enc-service.env + notification: ../../../../ansible/roles/stack-sunbird/templates/sunbird_notification-service.env + telemetry: ../../../../ansible/roles/stack-sunbird/templates/sunbird_telemetry-service.env + userorg: ../../../../ansible/roles/stack-sunbird/templates/sunbird_user-org-service.env + player: ../../../../ansible/roles/stack-sunbird/templates/sunbird_player.env + print: ../../../../ansible/roles/stack-sunbird/templates/sunbird_print-service.env + search: + - ../../../../ansible/roles/stack-sunbird/templates/search-service_application.conf + - ../../../../ansible/roles/stack-sunbird/templates/search-service_logback.xml + taxonomy: + - ../../../../ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf + - ../../../../ansible/roles/stack-sunbird/templates/taxonomy-service_logback.xml + report: ../../../../ansible/roles/stack-sunbird/templates/sunbird_report-service.env + telemetry-dp-logstash: ../../../../ansible/roles/stack-sunbird/templates/telemetry-logstash-datapipeline.conf + nodebb: ../../../../ansible/roles/stack-sunbird/templates/sunbird_nodebb.env + analytics: ../../../../ansible/roles/stack-sunbird/templates/analytics_api_service.conf + discussionsmw: ../../../../ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env + +sunbird_portal_player_cdn_enabled: false + +########### stack-apimanager defaults vars ##### for kubernetes ######### +kong_database: postgres +kong_ssl: true +kong_replicas: 1 +kong_reservation_memory: 64M +kong_limit_memory: 256M +kibana_oauth_authenticated_email_domains: [] + +##### livenessprobe and readinessprobe ####### +groups_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +lms_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +apimanager_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /status + port: 8001 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /status + port: 8001 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 1 + +assessment_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +cert_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +certregistry_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +content_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +enc_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 8013 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 8013 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +knowledgemw_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 5000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 5000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +learner_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +notification_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +player_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /service/health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 3000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +print_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 5000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +telemetry_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9001 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9001 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +userorg_liveness_readiness: + readinessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +discussionsmw_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 3002 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +search_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 5 + +taxonomy_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +search_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +taxonomy_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +report_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 3030 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /service/health + port: 3030 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 5 + +analytics_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +adminutil__access_keyprefix: "access" +adminutil__access_keystart: 0 +adminutil__access_keycount: 0 + +adminutil_access_values: +- key_to_register: access + values_to_pass: + basepath: "/keys/" + keyprefix: "{{ adminutil__access_keyprefix }}" + keystart: "{{ adminutil__access_keystart }}" + keycount: "{{ adminutil__access_keystart + adminutil__access_keycount - 1 }}" + role_to_run: + - decrypt.yml + - generate-keys.yml + - copy-to-helm-public.yml + +# analytics-service related vars +cassandra: + keyspace_prefix: "{{ env_name }}_" + +default_channel_id: "in.ekstep" +default_channel: "{{default_org_hash_id}}" +channel_data_exhaust_bucket: dev-data-store +metadata_redis_host: "{{ groups['dp-redis'][0] }}" +lp_composite_search_host: "{{ groups['composite-search-cluster'][0] }}" +ingestion_kafka_brokers: "{{groups['ingestion-cluster-kafka']|join(':9092,')}}:9092" +geo_location_city: "{{ env_name }}_geo_location_city" +geo_location_city_ipv4: "{{ env_name }}_geo_location_city_ipv4" +report_config: "{{ env_name }}_report_config" +job_request: "{{ env_name }}_job_request" +experiment_definition: "{{ env_name }}_experiment_definition" + +#### Exhuats API consumer +exhaust_api_consumer_ids: ["273f3b18-5dda-4a27-984a-060c7cd398d3"] # being used in analytics api to access the analytics exhaust api + +user_profile_read_url: "http://learner-service:9000/v1/user/read/" # being used in analytics api to get user details +org_search_url: "http://learner-service:9000/v1/org/search" # being used in analytics api to get mhrd tenant id +dataexhaust_super_admin_channel: "sunbird" # being used in analytics api for authorizing user with super admin channel diff --git a/ansible/roles/stack-sunbird/tasks/assessment-service.yml b/ansible/roles/stack-sunbird/tasks/assessment-service.yml new file mode 100644 index 000000000..282eb3ab9 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/assessment-service.yml @@ -0,0 +1,35 @@ +--- +- name: Remove assessment-service + shell: "docker service rm assessment-service" + ignore_errors: yes + +- name: Save configurations into an env file + become: yes + template: src=assessment-service_application.conf dest=/home/deployer/config/assessment-service.conf mode=0644 + +- name: Save logback configuration into a env file + become: yes + template: src=assessment-service_logback.xml dest=/home/deployer/config/assessment-service_logback.xml mode=0644 + +- name: Remove old assessment-service.conf docker config + become: yes + shell: "docker config rm assessment-service.conf" + ignore_errors: yes + +- name: Save as docker config + become: yes + shell: "docker config create assessment-service.conf /home/deployer/config/assessment-service.conf" + +- name: Remove old logback.xml docker config + become: yes + shell: "docker config rm assessment-service_logback.xml" + ignore_errors: yes + +- name: Copy new logback.xml + become: yes + shell: "docker config create assessment-service_logback.xml /home/deployer/config/assessment-service_logback.xml" + +- name: Deploy assessment-service + shell: "docker service create --with-registry-auth --replicas {{ assessment_service_replicas }} -p 9003:9000 --name assessment-service --hostname assessment-service --reserve-memory {{ assessment_service_reservation_memory }} --limit-memory {{ assessment_service_limit_memory }} --limit-cpu {{ assessment_service_limit_cpu }} --reserve-cpu {{ assessment_service_reservation_cpu }} --health-cmd 'wget -qO- assessment-service:9000/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --config source=assessment-service.conf,target=/home/sunbird/assessment-service-1.0-SNAPSHOT/config/application.conf,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/cert_registry_service.yml b/ansible/roles/stack-sunbird/tasks/cert_registry_service.yml index d3b772133..d23170bf2 100644 --- a/ansible/roles/stack-sunbird/tasks/cert_registry_service.yml +++ b/ansible/roles/stack-sunbird/tasks/cert_registry_service.yml @@ -4,4 +4,4 @@ ignore_errors: yes - name: Deploy cert registry service - shell: "docker service create --with-registry-auth --replicas {{ cert_registry_replicas }} -p 9013:9000 --name cert-registry-service --hostname cert-registry-service --limit-memory {{ cert_registry_limit_memory }} --limit-cpu {{ cert_registry_limit_cpu }} --health-cmd 'wget -qO- cert-registry-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_cert-registry-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ cert_registry_replicas }} -p 9013:9000 --name cert-registry-service --hostname cert-registry-service --limit-memory {{ cert_registry_limit_memory }} --limit-cpu {{ cert_registry_limit_cpu }} --reserve-memory {{ cert_registry_reserve_memory }} --reserve-cpu {{ cert_registry_reserve_cpu }} --health-cmd 'wget -qO- cert-registry-service:9000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_cert-registry-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" diff --git a/ansible/roles/stack-sunbird/tasks/cert_service.yml b/ansible/roles/stack-sunbird/tasks/cert_service.yml index f510b9ae6..144440462 100644 --- a/ansible/roles/stack-sunbird/tasks/cert_service.yml +++ b/ansible/roles/stack-sunbird/tasks/cert_service.yml @@ -4,5 +4,4 @@ ignore_errors: yes - name: Deploy cert service - shell: "docker service create --with-registry-auth --replicas {{ cert_replicas }} -p 9011:9000 --name cert-service --hostname cert-service --limit-memory {{ cert_limit_memory }} --limit-cpu {{ cert_limit_cpu }} --health-cmd 'wget -qO- cert-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_cert-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" - \ No newline at end of file + shell: "docker service create --with-registry-auth --replicas {{ cert_replicas }} -p 9011:9000 --name cert-service --hostname cert-service --limit-memory {{ cert_limit_memory }} --limit-cpu {{ cert_limit_cpu }} --reserve-memory {{ cert_reservation_memory }} --reserve-cpu {{ cert_reservation_cpu }} --health-cmd 'wget -qO- cert-service:9000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_cert-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" diff --git a/ansible/roles/stack-sunbird/tasks/common.yml b/ansible/roles/stack-sunbird/tasks/common.yml index 08b3abe7e..228aab159 100644 --- a/ansible/roles/stack-sunbird/tasks/common.yml +++ b/ansible/roles/stack-sunbird/tasks/common.yml @@ -32,5 +32,14 @@ - name: Remove env file file: path=/home/deployer/env/sunbird_{{service_name}}.env state=absent +- name: check if env file exists + stat: path={{role_path}}/templates/sunbird_{{service_name}}.env + register: out + delegate_to: "localhost" + +- debug: var=out + - name: Save service configurations into an env file template: src="sunbird_{{service_name}}.env" dest="/home/deployer/env/sunbird_{{service_name}}.env" mode=0644 + when: out.stat.exists + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/tasks/content-service.yml b/ansible/roles/stack-sunbird/tasks/content-service.yml new file mode 100644 index 000000000..6c113f186 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/content-service.yml @@ -0,0 +1,35 @@ +--- +- name: Remove content-service + shell: "docker service rm content-service" + ignore_errors: yes + +- name: Save configurations into an env file + become: yes + template: src=content-service_application.conf dest=/home/deployer/config/content-service.conf mode=0644 + +- name: Save logback configuration into a env file + become: yes + template: src=content-service_logback.xml dest=/home/deployer/config/content-service_logback.xml mode=0644 + +- name: Remove old application.conf docker config + become: yes + shell: "docker config rm content-service.conf" + ignore_errors: yes + +- name: Save as docker config + become: yes + shell: "docker config create content-service.conf /home/deployer/config/content-service.conf" + +- name: Remove old logback.xml docker config + become: yes + shell: "docker config rm content-service_logback.xml" + ignore_errors: yes + +- name: Copy new logback.xml + become: yes + shell: "docker config create content-service_logback.xml /home/deployer/config/content-service_logback.xml" + +- name: Deploy content-service + shell: "docker service create --with-registry-auth --replicas {{ content_service_replicas }} -p 9002:9000 --name content-service --hostname content-service --reserve-memory {{ content_service_reservation_memory }} --limit-memory {{ content_service_limit_memory }} --limit-cpu {{ content_service_limit_cpu }} --reserve-cpu {{ content_service_reservation_cpu }} --health-cmd 'wget -qO- content-service:9000/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env JAVA_OPTIONS={{ content_java_mem_limit }} --config source=content-service.conf,target=/home/sunbird/content-service-1.0-SNAPSHOT/config/application.conf,mode=0644 --config source=content-service_logback.xml,target=/home/sunbird/content-service-1.0-SNAPSHOT/config/logback.xml,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/druid_proxy_api_service.yml b/ansible/roles/stack-sunbird/tasks/druid_proxy_api_service.yml new file mode 100644 index 000000000..e428cdd7d --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/druid_proxy_api_service.yml @@ -0,0 +1,9 @@ +--- +- name: Remove DruidProxyAPI service + shell: "docker service rm druid-proxy-api" + ignore_errors: yes + +- name: Deploy DruidProxyAPI service + shell: "docker service create --replicas {{ druid_proxy_replicas }} -p 8082:8082 --name druid-proxy-api --hostname druid-proxy-api --reserve-memory {{ druid_proxy_reservation_memory }} --limit-memory {{ druid_proxy_limit_memory }} --network application_default --env-file /home/deployer/env/sunbird_druid-proxy-api.env --with-registry-auth {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/enc_service.yml b/ansible/roles/stack-sunbird/tasks/enc_service.yml index 1ca814001..31e2c1634 100644 --- a/ansible/roles/stack-sunbird/tasks/enc_service.yml +++ b/ansible/roles/stack-sunbird/tasks/enc_service.yml @@ -4,4 +4,4 @@ ignore_errors: yes - name: Deploy enc service - shell: "docker service create --with-registry-auth --replicas {{ enc_replicas }} -p 9010:8013 --name enc-service --hostname enc-service --limit-memory {{ enc_limit_memory }} --limit-cpu {{ enc_limit_cpu }} --health-cmd 'wget -qO- enc-service:8013/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_enc-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ enc_replicas }} -p 9010:8013 --name enc-service --hostname enc-service --limit-memory {{ enc_limit_memory }} --limit-cpu {{ enc_limit_cpu }} --reserve-cpu {{ enc_reservation_cpu }} --reserve-memory {{ enc_reservation_memory }} --health-cmd 'wget -qO- enc-service:8013/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_enc-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" diff --git a/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml b/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml index a855ca06d..343de7b0f 100644 --- a/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml +++ b/ansible/roles/stack-sunbird/tasks/knowledge-mw-service.yml @@ -4,6 +4,6 @@ ignore_errors: yes - name: Deploy knowledge-mw service - shell: "docker service create --with-registry-auth --replicas {{ content_replicas }} -p 5000:5000 --name knowledge-mw-service --hostname knowledge-mw-service --reserve-memory {{ content_reservation_memory }} --limit-memory {{ content_limit_memory }} --limit-cpu {{ content_limit_cpu }} --reserve-cpu {{ content_reservation_cpu }} --health-cmd 'wget -qO- knowledge-mw-service:5000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_knowledge-mw-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ knowledge_mw_service_replicas }} -p 5000:5000 --name knowledge-mw-service --hostname knowledge-mw-service --reserve-memory {{ knowledge_mw_service_reservation_memory }} --limit-memory {{ knowledge_mw_service_limit_memory }} --limit-cpu {{ knowledge_mw_service_limit_cpu }} --reserve-cpu {{ knowledge_mw_service_reservation_cpu }} --health-cmd 'wget -qO- knowledge-mw-service:5000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_knowledge-mw-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" args: chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/learner_service.yml b/ansible/roles/stack-sunbird/tasks/learner_service.yml index a8ce49222..3320ee97d 100644 --- a/ansible/roles/stack-sunbird/tasks/learner_service.yml +++ b/ansible/roles/stack-sunbird/tasks/learner_service.yml @@ -4,6 +4,6 @@ ignore_errors: yes - name: Deploy learner service - shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9000:9000 --name learner-service --hostname learner-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- learner-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_learner-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9000:9000 --name learner-service --hostname learner-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- learner-service:9000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env JAVA_OPTIONS={{ learner_java_mem_limit }} --env-file /home/deployer/env/sunbird_learner-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" args: chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/learning-service.yml b/ansible/roles/stack-sunbird/tasks/learning-service.yml deleted file mode 100644 index 4c2aa1bb4..000000000 --- a/ansible/roles/stack-sunbird/tasks/learning-service.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Remove learning service - shell: "docker service rm learning-service" - ignore_errors: yes - -- name: Deploy learning service - shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9696:9000 --name learning-service --hostname learning-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- learning-service:9000/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_learning-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" - args: - chdir: /home/deployer/stack \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/tasks/lms_service.yml b/ansible/roles/stack-sunbird/tasks/lms_service.yml index 5388bb281..1be9bba68 100644 --- a/ansible/roles/stack-sunbird/tasks/lms_service.yml +++ b/ansible/roles/stack-sunbird/tasks/lms_service.yml @@ -4,6 +4,6 @@ ignore_errors: yes - name: Deploy lms service - shell: "docker service create --with-registry-auth --replicas {{ learner_replicas }} -p 9005:9000 --name lms-service --hostname lms-service --reserve-memory {{ learner_reservation_memory }} --limit-memory {{ learner_limit_memory }} --limit-cpu {{ learner_limit_cpu }} --reserve-cpu {{ learner_reservation_cpu }} --health-cmd 'wget -qO- lms-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_lms-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ lms_service_replicas }} -p 9005:9000 --name lms-service --hostname lms-service --reserve-memory {{ lms_service_reservation_memory }} --limit-memory {{ lms_service_limit_memory }} --limit-cpu {{ lms_service_limit_cpu }} --reserve-cpu {{ lms_service_reservation_cpu }} --health-cmd 'wget -qO- lms-service:9000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env JAVA_OPTIONS={{ lms_java_mem_limit }} --env-file /home/deployer/env/sunbird_lms-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" args: chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/main.yml b/ansible/roles/stack-sunbird/tasks/main.yml index e691934b8..6e5c02554 100644 --- a/ansible/roles/stack-sunbird/tasks/main.yml +++ b/ansible/roles/stack-sunbird/tasks/main.yml @@ -40,5 +40,25 @@ - include: telemetry_logstash_datapipeline.yml when: deploy_telemetry_logstash_datapipeline is defined - - include: learning-service.yml - when: deploy_learning is defined + - include: content-service.yml + when: deploy_content is defined + + - include: assessment-service.yml + when: deploy_assessment is defined + + - include: druid_proxy_api_service.yml + when: deploy_druid_proxy_api is defined + + - include: print-service.yml + when: deploy_print is defined + + - include: report-service.yml + when: deploy_report is defined + + - include: search-service.yml + when: deploy_search is defined + + - include: taxonomy-service.yml + when: deploy_taxonomy is defined + + - include: restart_private_proxy.yml diff --git a/ansible/roles/stack-sunbird/tasks/notification_service.yml b/ansible/roles/stack-sunbird/tasks/notification_service.yml index 22f3b42d5..fe69e6e41 100644 --- a/ansible/roles/stack-sunbird/tasks/notification_service.yml +++ b/ansible/roles/stack-sunbird/tasks/notification_service.yml @@ -4,4 +4,4 @@ ignore_errors: yes - name: Deploy notification service - shell: "docker service create --with-registry-auth --replicas {{ notification_replicas }} -p 9012:9000 --name notification-service --hostname notification-service --limit-memory {{ notification_limit_memory }} --limit-cpu {{ notification_limit_cpu }} --network application_default --env-file /home/deployer/env/sunbird_notification-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ notification_replicas }} -p 9012:9000 --name notification-service --hostname notification-service --reserve-cpu {{ notification_reserve_cpu }} --reserve-memory {{ notification_reserve_memory }} --limit-memory {{ notification_limit_memory }} --limit-cpu {{ notification_limit_cpu }} --network application_default --env-file /home/deployer/env/sunbird_notification-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" diff --git a/ansible/roles/stack-sunbird/tasks/print-service.yml b/ansible/roles/stack-sunbird/tasks/print-service.yml new file mode 100644 index 000000000..2434863e5 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/print-service.yml @@ -0,0 +1,9 @@ +--- +- name: Remove print service + shell: "docker service rm print-service" + ignore_errors: yes + +- name: Deploy print service + shell: "docker service create --with-registry-auth --replicas {{ print_service_replicas }} -p 5001:5000 --name print-service --hostname print-service --reserve-memory {{ print_service_reservation_memory }} --limit-memory {{ print_service_limit_memory }} --limit-cpu {{ print_service_limit_cpu }} --reserve-cpu {{ print_service_reservation_cpu }} --health-cmd 'wget -qO- print-service:5000/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_print-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/report-service.yml b/ansible/roles/stack-sunbird/tasks/report-service.yml new file mode 100644 index 000000000..bd4397ae7 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/report-service.yml @@ -0,0 +1,9 @@ +--- +- name: Remove report service + shell: "docker service rm report-service" + ignore_errors: yes + +- name: Deploy report service + shell: "docker service create --replicas {{ report_replicas }} -p 3030:3030 --name report-service --hostname report-service --reserve-memory {{ report_reservation_memory }} --reserve-cpu {{ report_reservation_cpu }} --limit-memory {{ report_limit_memory }} --limit-cpu {{ report_limit_cpu }} --network application_default --env-file /home/deployer/env/sunbird_report-service.env --with-registry-auth {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/restart_private_proxy.yml b/ansible/roles/stack-sunbird/tasks/restart_private_proxy.yml new file mode 100644 index 000000000..8244b6e03 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/restart_private_proxy.yml @@ -0,0 +1,16 @@ +--- +- name: check if private proxy is running + shell: docker service ps private_proxy + ignore_errors: yes + register: private_proxy_status + +- name: Getting the list of proxy services host details + shell: "getent ahostsv4 $(docker service ps private_proxy | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | uniq" + register: proxy_details + when: private_proxy_status.failed != true + +- name: reload proxy containers configurations + shell: "docker ps | grep proxy_proxy | awk '{print $1}' | xargs -I % docker exec % nginx -s reload;" + delegate_to: "{{item}}" + with_items: "{{(proxy_details.stdout_lines)|list}}" + when: proxy_details.changed diff --git a/ansible/roles/stack-sunbird/tasks/search-service.yml b/ansible/roles/stack-sunbird/tasks/search-service.yml new file mode 100644 index 000000000..40184de5b --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/search-service.yml @@ -0,0 +1,35 @@ +--- +- name: Remove search-service + shell: "docker service rm search-service" + ignore_errors: yes + +- name: Save configurations into an env file + become: yes + template: src=search-service_application.conf dest=/home/deployer/config/search-service.conf mode=0644 + +- name: Save logback configuration into a env file + become: yes + template: src=search-service_logback.xml dest=/home/deployer/config/search-service_logback.xml mode=0644 + +- name: Remove old application.conf docker config + become: yes + shell: "docker config rm search-service.conf" + ignore_errors: yes + +- name: Save as docker config + become: yes + shell: "docker config create search-service.conf /home/deployer/config/search-service.conf" + +- name: Remove old logback.xml docker config + become: yes + shell: "docker config rm search-service_logback.xml" + ignore_errors: yes + +- name: Copy new logback.xml + become: yes + shell: "docker config create search-service_logback.xml /home/deployer/config/search-service_logback.xml" + +- name: Deploy search-service + shell: "docker service create --with-registry-auth --replicas {{ search_service_replicas }} -p 9004:9000 --name search-service --hostname search-service --reserve-memory {{ search_service_reservation_memory }} --limit-memory {{ search_service_limit_memory }} --limit-cpu {{ search_service_limit_cpu }} --reserve-cpu {{ search_service_reservation_cpu }} --health-cmd 'wget -qO- search-service:9000/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --config source=search-service.conf,target=/home/sunbird/search-service-1.0-SNAPSHOT/config/application.conf,mode=0644 --config source=search-service_logback.xml,target=/home/sunbird/search-service-1.0-SNAPSHOT/config/logback.xml,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/tasks/service_stack.yml b/ansible/roles/stack-sunbird/tasks/service_stack.yml index e8bb268d0..401fb2506 100644 --- a/ansible/roles/stack-sunbird/tasks/service_stack.yml +++ b/ansible/roles/stack-sunbird/tasks/service_stack.yml @@ -39,7 +39,7 @@ seconds: 40 - name: Getting the list of proxy services host details - shell: "getent hosts $(docker service ps proxy_proxy | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | grep -v f " + shell: "getent ahostsv4 $(docker service ps proxy_proxy | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | uniq" register: proxy_details - name: reload proxy containers configurations @@ -47,3 +47,13 @@ delegate_to: "{{item}}" with_items: "{{(proxy_details.stdout_lines)|list}}" register: proxy_out + +- name: Getting the list of kong services host details + shell: "getent ahostsv4 $(docker service ps api-manager_kong | grep Runn | awk '{print $4}') | cut -d \" \" -f1 | uniq" + register: kong_details + +- name: reload kong container configurations + shell: "docker ps | grep api-manager_kong | awk '{print $1}' | xargs -I % docker exec % kong reload;" + delegate_to: "{{item}}" + with_items: "{{(kong_details.stdout_lines)|list}}" + register: kong_out diff --git a/ansible/roles/stack-sunbird/tasks/taxonomy-service.yml b/ansible/roles/stack-sunbird/tasks/taxonomy-service.yml new file mode 100644 index 000000000..c2f4fa841 --- /dev/null +++ b/ansible/roles/stack-sunbird/tasks/taxonomy-service.yml @@ -0,0 +1,35 @@ +--- +- name: Remove taxonomy-service + shell: "docker service rm taxonomy-service" + ignore_errors: yes + +- name: Save configurations into an env file + become: yes + template: src=taxonomy-service_application.conf dest=/home/deployer/config/taxonomy-service.conf mode=0644 + +- name: Save logback configuration into a env file + become: yes + template: src=taxonomy-service_logback.xml dest=/home/deployer/config/taxonomy-service_logback.xml mode=0644 + +- name: Remove old application.conf docker config + become: yes + shell: "docker config rm taxonomy-service.conf" + ignore_errors: yes + +- name: Save as docker config + become: yes + shell: "docker config create taxonomy-service.conf /home/deployer/config/taxonomy-service.conf" + +- name: Remove old logback.xml docker config + become: yes + shell: "docker config rm taxonomy-service_logback.xml" + ignore_errors: yes + +- name: Copy new logback.xml + become: yes + shell: "docker config create taxonomy-service_logback.xml /home/deployer/config/taxonomy-service_logback.xml" + +- name: Deploy taxonomy-service + shell: "docker service create --with-registry-auth --replicas {{ taxonomy_service_replicas }} -p 9006:9000 --name taxonomy-service --hostname taxonomy-service --reserve-memory {{ taxonomy_service_reservation_memory }} --limit-memory {{ taxonomy_service_limit_memory }} --limit-cpu {{ taxonomy_service_limit_cpu }} --reserve-cpu {{ taxonomy_service_reservation_cpu }} --health-cmd 'wget -qO- taxonomy-service:9000/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --config source=taxonomy-service.conf,target=/home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/application.conf,mode=0644 --config source=taxonomy-service_logback.xml,target=/home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/logback.xml,mode=0644 {{hub_org}}/{{image_name}}:{{image_tag}}" + args: + chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/telemetry_service.yml b/ansible/roles/stack-sunbird/tasks/telemetry_service.yml index 2fec63d0c..073b85f94 100644 --- a/ansible/roles/stack-sunbird/tasks/telemetry_service.yml +++ b/ansible/roles/stack-sunbird/tasks/telemetry_service.yml @@ -4,6 +4,6 @@ ignore_errors: yes - name: Deploy telemetry service - shell: "docker service create --replicas {{ telemetry_replicas }} -p 9001:9001 --name telemetry-service --hostname telemetry-service --reserve-memory {{ telemetry_reservation_memory }} --limit-memory {{ telemetry_limit_memory }} --network application_default --env-file /home/deployer/env/sunbird_telemetry-service.env --with-registry-auth {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --replicas {{ telemetry_replicas }} -p 9001:9001 --name telemetry-service --hostname telemetry-service --reserve-cpu {{ telemetry_reservation_cpu }} --reserve-memory {{ telemetry_reservation_memory }} --limit-memory {{ telemetry_limit_memory }} --limit-cpu {{ telemetry_limit_cpu }} --network application_default --env-file /home/deployer/env/sunbird_telemetry-service.env --with-registry-auth {{hub_org}}/{{image_name}}:{{image_tag}}" args: chdir: /home/deployer/stack diff --git a/ansible/roles/stack-sunbird/tasks/user_org_service.yml b/ansible/roles/stack-sunbird/tasks/user_org_service.yml index 24ab18ebe..52accff45 100644 --- a/ansible/roles/stack-sunbird/tasks/user_org_service.yml +++ b/ansible/roles/stack-sunbird/tasks/user_org_service.yml @@ -4,4 +4,4 @@ ignore_errors: yes - name: Deploy user org service - shell: "docker service create --with-registry-auth --replicas {{ user_org_replicas }} -p 9009:9000 --name user-org-service --hostname user-org-service --limit-memory {{ user_org_limit_memory }} --limit-cpu {{ user_org_limit_cpu }} --health-cmd 'wget -qO- user-org-service:9000/service/health || exit 1' --health-timeout 3s --health-retries 3 --network application_default --env-file /home/deployer/env/sunbird_user-org-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" + shell: "docker service create --with-registry-auth --replicas {{ user_org_replicas }} -p 9009:9000 --name user-org-service --hostname user-org-service --limit-memory {{ user_org_limit_memory }} --limit-cpu {{ user_org_limit_cpu }} --health-cmd 'wget -qO- user-org-service:9000/service/health || exit 1' --health-timeout 10s --health-retries 5 --network application_default --env-file /home/deployer/env/sunbird_user-org-service.env {{hub_org}}/{{image_name}}:{{image_tag}}" diff --git a/ansible/roles/stack-sunbird/templates/analytics_api_service.conf b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf new file mode 100644 index 000000000..5007c9b62 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/analytics_api_service.conf @@ -0,0 +1,221 @@ +application.env="{{ env_name }}" +spark.cassandra.connection.host={{groups['dp-cassandra'][0]}} +spark.cassandra.connection.port="9042" +# Data Exhaust API +data_exhaust.list.limit="100" +data_exhaust.retry.limit="3" +data_exhaust.dataset.list=["eks-consumption-raw", "eks-consumption-summary", "eks-consumption-metrics","eks-creation-raw", "eks-creation-summary", "eks-creation-metrics"] +data_exhaust.dataset.default="eks-consumption-raw" +data_exhaust.output_format="json" +data_exhaust.bucket="reports" +cassandra.service.embedded.enable=false +cassandra.keyspace_prefix="{{ cassandra.keyspace_prefix }}" +device-register-controller-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 3.0 + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 4 + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 8 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 +} +device-register-actor-dispatcher { + type = "Dispatcher" + executor = "thread-pool-executor" + thread-pool-executor { + fixed-pool-size = 8 + } + throughput = 1 +} +device-profile-actor-dispatcher { + type = "Dispatcher" + executor = "thread-pool-executor" + thread-pool-executor { + fixed-pool-size = 8 + } + throughput = 1 +} +experiment-actor { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 3.0 + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 16 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 +} +default-dispatcher { + executor = "fork-join-executor" + fork-join-executor { + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 3.0 + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 16 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 +} +#AKKA Configuration +akka { + actor { + deployment { + /druid-health-actor { + router = smallest-mailbox-pool + nr-of-instances = 10 + } + /job-service-actor { + router = smallest-mailbox-pool + nr-of-instances = 10 + } + /device-register-actor { + router = smallest-mailbox-pool + dispatcher = device-register-actor-dispatcher + nr-of-instances = 6 + } + /device-profile-actor { + router = smallest-mailbox-pool + dispatcher = device-profile-actor-dispatcher + nr-of-instances = 6 + } + /client-log-actor { + router = smallest-mailbox-pool + nr-of-instances = 4 + } + /experiment-actor { + router = smallest-mailbox-pool + nr-of-instances = 2 + } + } + } +} +#Netty Configuration +play.server { + # The server provider class name + provider = "play.core.server.NettyServerProvider" + netty { + # The number of event loop threads. 0 means let Netty decide, which by default will select 2 times the number of + # available processors. + eventLoopThreads = 30 + # Whether the Netty wire should be logged + log.wire = true + # The transport to use, either jdk or native. + # Native socket transport has higher performance and produces less garbage but are only available on linux + transport = "native" + } +} +play.modules.enabled+="modules.ActorInjector" +play.filters.enabled+="filter.RequestInterceptor" +play.filters.enabled+="play.filters.hosts.AllowedHostsFilter" +play.filters.disabled+="play.filters.csrf.CSRFFilter" +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# If you deploy your application to several instances be sure to use the same key! +play.http.secret.key="{{dp_play_http_secret_key}}" +play.filters.hosts { + # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com") + # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot. + # "." matches all domains, and "" matches an empty or nonexistent host. + allowed = ["."] +} +# body parser +play.http.parser.maxMemoryBuffer=10M +default.consumption.app.id="no_value" +default.channel.id="{{default_channel_id}}" +default.creation.app.id="no_value" +postgres.db="{{postgres.db_name}}" +postgres.url="jdbc:postgresql://{{ postgres.db_url }}:5432/" +postgres.user="{{ postgres.db_username }}" +postgres.pass="{{ postgres.db_password }}" +postgres.table_name="{{ postgres.db_table_name }}" +postgres.table.geo_location_city.name="{{ geo_location_city }}" +postgres.table.geo_location_city_ipv4.name="{{ geo_location_city_ipv4 }}" +postgres.table.report_config.name="{{ report_config }}" +postgres.table.job_request.name="{{ job_request }}" +postgres.table.experiment_definition.name="{{ experiment_definition }}" +default.channel="{{ default_channel }}" +elasticsearch.service.endpoint="http://{{groups['composite-search-cluster'][0]}}:9200" +elasticsearch.index.compositesearch.name="compositesearch" +elasticsearch.index.dialcodemetrics.name="dialcodemetrics" +metrics.dialcodemetrics.request.limit=1000 +dataexhaust.authorization_check=true +user.profile.url="{{ user_profile_read_url }}" +org.search.url="{{ org_search_url }}" +standard.dataexhaust.roles=["ORG_ADMIN","REPORT_ADMIN"] +ondemand.dataexhaust.roles=["ORG_ADMIN","REPORT_ADMIN","CONTENT_CREATOR","COURSE_MENTOR"] +dataexhaust.super.admin.channel="{{ dataexhaust_super_admin_channel }}" +storage-service.request-signature-version="AWS4-HMAC-SHA256" +s3service.region="ap-south-1" +#channel exhaust configs +channel { + data_exhaust { + whitelisted.consumers={{ exhaust_api_consumer_ids }} + expiryMins = 30 + dataset { + default { + bucket = "{{channel_data_exhaust_bucket}}" + basePrefix = "data-exhaust/" + } + raw { + bucket = "{{channel_data_exhaust_bucket}}" + basePrefix = "data-exhaust/" + } + summary { + bucket = "{{channel_data_exhaust_bucket}}" + basePrefix = "data-exhaust/" + } + summary-rollup { + bucket = "{{channel_data_exhaust_bucket}}" + basePrefix = "data-exhaust/" + } + } + } +} +cloud_storage_type="azure" +storage.key.config="azure_storage_key" +storage.secret.config="azure_storage_secret" +metrics.time.interval.min=30 +cache.refresh.time.interval.min=1440 +redis.host="{{metadata_redis_host}}" +#redis.host="localhost" +redis.port=6379 +#redis.port=__redis_port__ +redis.connection.max=20 +redis.connection.idle.max=20 +redis.connection.idle.min=10 +redis.connection.minEvictableIdleTimeSeconds=120 +redis.connection.timeBetweenEvictionRunsSeconds=300 +elasticsearch.host="{{lp_composite_search_host}}" +elasticsearch.port=9200 +# experiment service settings +elasticsearch.searchExperiment.index="experiment" +elasticsearch.searchExperiment.fieldWeight="{\"userId\":3.0, \"deviceId\":3.0, \"url\":3.0 }" +elasticsearch.searchExperiment.matchQueryScore=9.0 +deviceRegisterAPI.experiment.enable=false +experimentService.redisEmptyValueExpirySeconds=86400 +redis.experimentIndex=9 +redis.deviceIndex=2 +druid.coordinator.host="http://{{groups['raw-coordinator'][0]}}:8081/" +druid.healthcheck.url="druid/coordinator/v1/loadstatus" +device.api.enable.debug.log=true +kafka.device.register.topic={{ env_name }}.events.deviceprofile +kafka.metrics.event.topic={{ env_name }}.pipeline_metrics +kafka.broker.list="{{ ingestion_kafka_brokers }}" diff --git a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf new file mode 100644 index 000000000..7685cd4ea --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf @@ -0,0 +1,381 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ +# Play uses HOCON as its configuration file format. HOCON has a number +# of advantages over other config formats, but there are two things that +# can be used when modifying settings. +# +# You can include other configuration files in this main application.conf file: +#include "extra-config.conf" +# +# You can declare variables and substitute for them: +#mykey = ${some.value} +# +# And if an environment variable exists when there is no other substitution, then +# HOCON will fall back to substituting environment variable: +#mykey = ${JAVA_HOME} + +## Akka +# https://www.playframework.com/documentation/latest/ScalaAkka#Configuration +# https://www.playframework.com/documentation/latest/JavaAkka#Configuration +# ~~~~~ +# Play uses Akka internally and exposes Akka Streams and actors in Websockets and +# other streaming HTTP responses. +akka { + # "akka.log-config-on-start" is extraordinarly useful because it log the complete + # configuration at INFO level, including defaults and overrides, so it s worth + # putting at the very top. + # + # Put the following in your conf/logback.xml file: + # + # + # + # And then uncomment this line to debug the configuration. + # + #log-config-on-start = true + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + actors-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + actor { + deployment { + /healthActor + { + router = smallest-mailbox-pool + nr-of-instances = 5 + dispatcher = actors-dispatcher + } + /itemSetActor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = actors-dispatcher + } + } + } +} + +## Secret key +# http://www.playframework.com/documentation/latest/ApplicationSecret +# ~~~~~ +# The secret key is used to sign Play's session cookie. +# This must be changed for production, but we don't recommend you change it in this file. +play.http.secret.key="{{ graph_passport_key }}" + +## Modules +# https://www.playframework.com/documentation/latest/Modules +# ~~~~~ +# Control which modules are loaded when Play starts. Note that modules are +# the replacement for "GlobalSettings", which are deprecated in 2.5.x. +# Please see https://www.playframework.com/documentation/latest/GlobalSettings +# for more information. +# +# You can also extend Play functionality by using one of the publically available +# Play modules: https://playframework.com/documentation/latest/ModuleDirectory +play.modules { + # By default, Play will load any class called Module that is defined + # in the root package (the "app" directory), or you can define them + # explicitly below. + # If there are any built-in modules that you want to enable, you can list them here. + #enabled += my.application.Module + + # If there are any built-in modules that you want to disable, you can list them here. + #disabled += "" + enabled += modules.AssessmentModule +} + +## IDE +# https://www.playframework.com/documentation/latest/IDE +# ~~~~~ +# Depending on your IDE, you can add a hyperlink for errors that will jump you +# directly to the code location in the IDE in dev mode. The following line makes +# use of the IntelliJ IDEA REST interface: +#play.editor="http://localhost:63342/api/file/?file=%s&line=%s" + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# https://www.playframework.com/documentation/latest/ScalaI18N +# ~~~~~ +# Play comes with its own i18n settings, which allow the user's preferred language +# to map through to internal messages, or allow the language to be stored in a cookie. +play.i18n { + # The application languages + langs = [ "en" ] + + # Whether the language cookie should be secure or not + #langCookieSecure = true + + # Whether the HTTP only attribute of the cookie should be set to true + #langCookieHttpOnly = true +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + + ## Action Creator + # https://www.playframework.com/documentation/latest/JavaActionCreator + # ~~~~~ + #actionCreator = null + + ## ErrorHandler + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # If null, will attempt to load a class called ErrorHandler in the root package, + #errorHandler = null + + ## Session & Flash + # https://www.playframework.com/documentation/latest/JavaSessionFlash + # https://www.playframework.com/documentation/latest/ScalaSessionFlash + # ~~~~~ + session { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + + # Sets the max-age field of the cookie to 5 minutes. + # NOTE: this only sets when the browser will discard the cookie. Play will consider any + # cookie value with a valid signature to be a valid session forever. To implement a server side session timeout, + # you need to put a timestamp in the session and check it at regular intervals to possibly expire it. + #maxAge = 300 + + # Sets the domain on the session cookie. + #domain = "example.com" + } + + flash { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + } +} + +play.server.http.idleTimeout = 60s +play.http.parser.maxDiskBuffer = 10MB +parsers.anyContent.maxLength = 10MB + +## Netty Provider +# https://www.playframework.com/documentation/latest/SettingsNetty +# ~~~~~ +play.server.netty { + # Whether the Netty wire should be logged + log.wire = true + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + transport = "native" +} + +## WS (HTTP Client) +# https://www.playframework.com/documentation/latest/ScalaWS#Configuring-WS +# ~~~~~ +# The HTTP client primarily used for REST APIs. The default client can be +# configured directly, but you can also create different client instances +# with customized settings. You must enable this by adding to build.sbt: +# +# libraryDependencies += ws // or javaWs if using java +# +play.ws { + # Sets HTTP requests not to follow 302 requests + #followRedirects = false + + # Sets the maximum number of open HTTP connections for the client. + #ahc.maxConnectionsTotal = 50 + + ## WS SSL + # https://www.playframework.com/documentation/latest/WsSSL + # ~~~~~ + ssl { + # Configuring HTTPS with Play WS does not require programming. You can + # set up both trustManager and keyManager for mutual authentication, and + # turn on JSSE debugging in development with a reload. + #debug.handshake = true + #trustManager = { + # stores = [ + # { type = "JKS", path = "exampletrust.jks" } + # ] + #} + } +} + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +# libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +## Filter Configuration +# https://www.playframework.com/documentation/latest/Filters +# ~~~~~ +# There are a number of built-in filters that can be enabled and configured +# to give Play greater security. +# +play.filters { + + # Enabled filters are run automatically against Play. + # CSRFFilter, AllowedHostFilters, and SecurityHeadersFilters are enabled by default. + enabled = [filters.AccessLogFilter] + + # Disabled filters remove elements from the enabled list. + # disabled += filters.CSRFFilter + + + ## CORS filter configuration + # https://www.playframework.com/documentation/latest/CorsFilter + # ~~~~~ + # CORS is a protocol that allows web applications to make requests from the browser + # across different domains. + # NOTE: You MUST apply the CORS configuration before the CSRF filter, as CSRF has + # dependencies on CORS settings. + cors { + # Filter paths by a whitelist of path prefixes + #pathPrefixes = ["/some/path", ...] + + # The allowed origins. If null, all origins are allowed. + #allowedOrigins = ["http://www.example.com"] + + # The allowed HTTP methods. If null, all methods are allowed + #allowedHttpMethods = ["GET", "POST"] + } + + ## Security headers filter configuration + # https://www.playframework.com/documentation/latest/SecurityHeaders + # ~~~~~ + # Defines security headers that prevent XSS attacks. + # If enabled, then all options are set to the below configuration by default: + headers { + # The X-Frame-Options header. If null, the header is not set. + #frameOptions = "DENY" + + # The X-XSS-Protection header. If null, the header is not set. + #xssProtection = "1; mode=block" + + # The X-Content-Type-Options header. If null, the header is not set. + #contentTypeOptions = "nosniff" + + # The X-Permitted-Cross-Domain-Policies header. If null, the header is not set. + #permittedCrossDomainPolicies = "master-only" + + # The Content-Security-Policy header. If null, the header is not set. + #contentSecurityPolicy = "default-src 'self'" + } + + ## Allowed hosts filter configuration + # https://www.playframework.com/documentation/latest/AllowedHostsFilter + # ~~~~~ + # Play provides a filter that lets you configure which hosts can access your application. + # This is useful to prevent cache poisoning attacks. + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + #allowed = [".example.com", "localhost:9000"] + } +} + +play.http.parser.maxMemoryBuffer = 50MB +akka.http.parsing.max-content-length = 50MB +schema.base_path="{{ kp_schema_base_path | default('/home/sunbird/assessment-service-1.0-SNAPSHOT/schemas')}}" + +# Cassandra Configuration +cassandra.lp.connection="{{ lp_cassandra_connection }}" +content.keyspace = "{{ lp_cassandra_keyspace_prefix }}_content_store" + +# Redis Configuration +redis.host="{{ sunbird_lp_redis_host }}" +redis.port=6379 +redis.maxConnections=128 + +# Configuration +graph.dir=/data/graphDB +akka.request_timeout=30 +environment.id={{ environment_id }} +graph.ids=["domain"] +graph.passport.key.base="{{ graph_passport_key }}" +route.domain="{{ learning_neo4j_bolt_url }}" +route.bolt.write.domain="{{ learning_write_elb_url }}" +route.bolt.read.domain="{{ learning_read_elb_url }}" +route.all="{{ language_neo4j_bolt_url }}" +route.bolt.write.all="{{ language_write_elb_url }}" +route.bolt.read.all="{{ language_read_elb_url }}" + +shard.id={{ mw_shard_id }} +platform.auth.check.enabled=false +platform.cache.ttl=3600000 + +#Top N Config for Search Telemetry +telemetry_env=dev + +installation.id=ekstep + + +languageCode { + assamese : "as" + bengali : "bn" + english : "en" + gujarati : "gu" + hindi : "hi" + kannada : "ka" + marathi : "mr" + odia : "or" + tamil : "ta" + telugu : "te" +} + +kafka { + urls : "{{ kafka_urls }}" + topic.send.enable : true + topics.instruction : "{{ env_name }}.learning.job.request" +} +objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" +question.keyspace="{{ lp_cassandra_keyspace_prefix }}_question_store" +questionset.keyspace="{{ lp_cassandra_keyspace_prefix }}_hierarchy_store" diff --git a/ansible/roles/stack-sunbird/templates/assessment-service_logback.xml b/ansible/roles/stack-sunbird/templates/assessment-service_logback.xml new file mode 100644 index 000000000..1d3fdd0c6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/assessment-service_logback.xml @@ -0,0 +1,60 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + + + + + + + + + + %msg + + + {{env_name}}.telemetry.raw + + + + + + + + + bootstrap.servers={{lp_kafka_url}} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/content-service_application.conf b/ansible/roles/stack-sunbird/templates/content-service_application.conf new file mode 100644 index 000000000..c48ecb69b --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/content-service_application.conf @@ -0,0 +1,579 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ +# Play uses HOCON as its configuration file format. HOCON has a number +# of advantages over other config formats, but there are two things that +# can be used when modifying settings. +# +# You can include other configuration files in this main application.conf file: +#include "extra-config.conf" +# +# You can declare variables and substitute for them: +#mykey = ${some.value} +# +# And if an environment variable exists when there is no other substitution, then +# HOCON will fall back to substituting environment variable: +#mykey = ${JAVA_HOME} + +## Akka +# https://www.playframework.com/documentation/latest/ScalaAkka#Configuration +# https://www.playframework.com/documentation/latest/JavaAkka#Configuration +# ~~~~~ +# Play uses Akka internally and exposes Akka Streams and actors in Websockets and +# other streaming HTTP responses. +akka { + # "akka.log-config-on-start" is extraordinarly useful because it log the complete + # configuration at INFO level, including defaults and overrides, so it s worth + # putting at the very top. + # + # Put the following in your conf/logback.xml file: + # + # + # + # And then uncomment this line to debug the configuration. + # + #log-config-on-start = true + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + actors-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + actor { + deployment { + /contentActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + /channelActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + /collectionActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + /healthActor + { + router = smallest-mailbox-pool + nr-of-instances = 5 + dispatcher = actors-dispatcher + } + /licenseActor + { + router = smallest-mailbox-pool + nr-of-instances = 2 + dispatcher = actors-dispatcher + } + } + } +} + +## Secret key +# http://www.playframework.com/documentation/latest/ApplicationSecret +# ~~~~~ +# The secret key is used to sign Play's session cookie. +# This must be changed for production, but we don't recommend you change it in this file. +play.http.secret.key="{{ graph_passport_key }}" + +## Modules +# https://www.playframework.com/documentation/latest/Modules +# ~~~~~ +# Control which modules are loaded when Play starts. Note that modules are +# the replacement for "GlobalSettings", which are deprecated in 2.5.x. +# Please see https://www.playframework.com/documentation/latest/GlobalSettings +# for more information. +# +# You can also extend Play functionality by using one of the publically available +# Play modules: https://playframework.com/documentation/latest/ModuleDirectory +play.modules { + # By default, Play will load any class called Module that is defined + # in the root package (the "app" directory), or you can define them + # explicitly below. + # If there are any built-in modules that you want to enable, you can list them here. + enabled += modules.ContentModule + + # If there are any built-in modules that you want to disable, you can list them here. + #disabled += "" +} + +## IDE +# https://www.playframework.com/documentation/latest/IDE +# ~~~~~ +# Depending on your IDE, you can add a hyperlink for errors that will jump you +# directly to the code location in the IDE in dev mode. The following line makes +# use of the IntelliJ IDEA REST interface: +#play.editor="http://localhost:63342/api/file/?file=%s&line=%s" + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# https://www.playframework.com/documentation/latest/ScalaI18N +# ~~~~~ +# Play comes with its own i18n settings, which allow the user's preferred language +# to map through to internal messages, or allow the language to be stored in a cookie. +play.i18n { + # The application languages + langs = [ "en" ] + + # Whether the language cookie should be secure or not + #langCookieSecure = true + + # Whether the HTTP only attribute of the cookie should be set to true + #langCookieHttpOnly = true +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + + ## Action Creator + # https://www.playframework.com/documentation/latest/JavaActionCreator + # ~~~~~ + #actionCreator = null + + ## ErrorHandler + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # If null, will attempt to load a class called ErrorHandler in the root package, + #errorHandler = null + + ## Session & Flash + # https://www.playframework.com/documentation/latest/JavaSessionFlash + # https://www.playframework.com/documentation/latest/ScalaSessionFlash + # ~~~~~ + session { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + + # Sets the max-age field of the cookie to 5 minutes. + # NOTE: this only sets when the browser will discard the cookie. Play will consider any + # cookie value with a valid signature to be a valid session forever. To implement a server side session timeout, + # you need to put a timestamp in the session and check it at regular intervals to possibly expire it. + #maxAge = 300 + + # Sets the domain on the session cookie. + #domain = "example.com" + } + + flash { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + } +} + +play.server.http.idleTimeout = 60s +play.http.parser.maxDiskBuffer = 100MB +parsers.anyContent.maxLength = 100MB + +## Netty Provider +# https://www.playframework.com/documentation/latest/SettingsNetty +# ~~~~~ +play.server.netty { + # Whether the Netty wire should be logged + log.wire = true + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + transport = "native" +} + +## WS (HTTP Client) +# https://www.playframework.com/documentation/latest/ScalaWS#Configuring-WS +# ~~~~~ +# The HTTP client primarily used for REST APIs. The default client can be +# configured directly, but you can also create different client instances +# with customized settings. You must enable this by adding to build.sbt: +# +# libraryDependencies += ws // or javaWs if using java +# +play.ws { + # Sets HTTP requests not to follow 302 requests + #followRedirects = false + + # Sets the maximum number of open HTTP connections for the client. + #ahc.maxConnectionsTotal = 50 + + ## WS SSL + # https://www.playframework.com/documentation/latest/WsSSL + # ~~~~~ + ssl { + # Configuring HTTPS with Play WS does not require programming. You can + # set up both trustManager and keyManager for mutual authentication, and + # turn on JSSE debugging in development with a reload. + #debug.handshake = true + #trustManager = { + # stores = [ + # { type = "JKS", path = "exampletrust.jks" } + # ] + #} + } +} + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +# libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +## Filter Configuration +# https://www.playframework.com/documentation/latest/Filters +# ~~~~~ +# There are a number of built-in filters that can be enabled and configured +# to give Play greater security. +# +play.filters { + + # Enabled filters are run automatically against Play. + # CSRFFilter, AllowedHostFilters, and SecurityHeadersFilters are enabled by default. + enabled = [filters.AccessLogFilter] + + # Disabled filters remove elements from the enabled list. + # disabled += filters.CSRFFilter + + + ## CORS filter configuration + # https://www.playframework.com/documentation/latest/CorsFilter + # ~~~~~ + # CORS is a protocol that allows web applications to make requests from the browser + # across different domains. + # NOTE: You MUST apply the CORS configuration before the CSRF filter, as CSRF has + # dependencies on CORS settings. + cors { + # Filter paths by a whitelist of path prefixes + #pathPrefixes = ["/some/path", ...] + + # The allowed origins. If null, all origins are allowed. + #allowedOrigins = ["http://www.example.com"] + + # The allowed HTTP methods. If null, all methods are allowed + #allowedHttpMethods = ["GET", "POST"] + } + + ## Security headers filter configuration + # https://www.playframework.com/documentation/latest/SecurityHeaders + # ~~~~~ + # Defines security headers that prevent XSS attacks. + # If enabled, then all options are set to the below configuration by default: + headers { + # The X-Frame-Options header. If null, the header is not set. + #frameOptions = "DENY" + + # The X-XSS-Protection header. If null, the header is not set. + #xssProtection = "1; mode=block" + + # The X-Content-Type-Options header. If null, the header is not set. + #contentTypeOptions = "nosniff" + + # The X-Permitted-Cross-Domain-Policies header. If null, the header is not set. + #permittedCrossDomainPolicies = "master-only" + + # The Content-Security-Policy header. If null, the header is not set. + #contentSecurityPolicy = "default-src 'self'" + } + + ## Allowed hosts filter configuration + # https://www.playframework.com/documentation/latest/AllowedHostsFilter + # ~~~~~ + # Play provides a filter that lets you configure which hosts can access your application. + # This is useful to prevent cache poisoning attacks. + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + #allowed = [".example.com", "localhost:9000"] + } +} + +play.http.parser.maxMemoryBuffer = 50MB +akka.http.parsing.max-content-length = 50MB +schema.base_path="{{kp_schema_base_path | default('/home/sunbird/content-service-1.0-SNAPSHOT/schemas')}}" + +# Cassandra Configuration +cassandra { + lp { + connection: "{{ lp_cassandra_connection }}" + } + lpa { + connection: "{{ dp_cassandra_connection }}" + } +} + +collection { + keyspace: "{{ lp_cassandra_keyspace_prefix }}_hierarchy_store" + cache.enable: true + image.migration.enabled: true +} + +content { + keyspace: "{{ lp_cassandra_keyspace_prefix }}_content_store" + cache { + ttl: 86400 + enable: true + } + hierarchy { + removed_props_for_leafNodes: ["collections", "children", "usedByContent", "item_sets", "methods", "libraries", "editorState"] + } + # Added for supporting backward compatibility - remove in release-2.7.0 + tagging { + backward_enable: true + property: "subject,medium" + } + h5p { + library { + path: "{{ h5p_library_path | default('https://sunbirddev.blob.core.windows.net/sunbird-content-dev/h5p-standalone-1.3.4.zip') }}" + } + } + copy { + invalid_statusList: ["Flagged","FlaggedDraft","FraggedReview","Retired", "Processing"] + origin_data: ["name", "author", "license", "organisation"] + props_to_remove: ["downloadUrl", "artifactUrl", "variants", "createdOn", "collections", "children", "lastUpdatedOn", "SYS_INTERNAL_LAST_UPDATED_ON", "versionKey", "s3Key", "status", "pkgVersion", "toc_url", "mimeTypesCount", "contentTypesCount", "leafNodesCount", "childNodes", "prevState", "lastPublishedOn", "flagReasons", "compatibilityLevel", "size", "publishChecklist", "publishComment", "LastPublishedBy", "rejectReasons", "rejectComment", "gradeLevel", "subject", "medium", "board", "topic", "purpose", "subtopic", "contentCredits", "owner", "collaborators", "creators", "contributors", "badgeAssertions", "dialcodes", "concepts", "keywords", "reservedDialcodes", "dialcodeRequired", "leafNodes", "sYS_INTERNAL_LAST_UPDATED_ON", "prevStatus", "lastPublishedBy", "streamingUrl"] + } + media { + base_url: "{{content_media_base_url | default(proto + '://' + domain_name)}}" + } +} + +# Redis Configuration +redis { + host: "{{ sunbird_lp_redis_host }}" + port: 6379 + maxConnections: 128 +} + + +#--Maximum Content Package File Size Limit in Bytes (50 MB) +MAX_CONTENT_PACKAGE_FILE_SIZE_LIMIT=52428800 + +#--Maximum Asset File Size Limit in Bytes (50 MB) +MAX_ASSET_FILE_SIZE_LIMIT=52428800 + +#--No of Retry While File Download Fails +RETRY_ASSET_DOWNLOAD_COUNT=1 + +#Current environment - need this property post cloud-store implementation +cloud_storage { + env: {{ env }} + content.folder: "content" + asset.folder: "assets" + artefact.folder: "artifact" + bundle.folder: "bundle" + media.folder: "media" + ecar.folder: "ecar_files" + upload.url.ttl: "54000" +} + +# Configuration +akka.request_timeout: 30 +environment.id: {{ environment_id }} +graph { + dir: "/data/graphDB" + ids: ["domain"] + passport.key.base: "{{ graph_passport_key }}" +} +route { + domain: "{{ learning_neo4j_bolt_url }}" + all: "{{ language_neo4j_bolt_url }}" + bolt { + write { + domain: "{{ learning_write_elb_url }}" + all: "{{ language_write_elb_url }}" + }, + read { + domain: "{{ learning_read_elb_url }}" + all: "{{ language_read_elb_url }}" + } + } +} + +shard.id: {{ mw_shard_id }} +platform { + auth.check.enabled: false + cache.ttl: 3600000 + language.codes: ["as","bn","en","gu","hi","hoc","jun","ka","mai","mr","unx","or","san","sat","ta","te","urd", "pj"] +} + +#Top N Config for Search Telemetry +telemetry_env: {{ env }} + +installation.id: ekstep + +channel { + default: "in.ekstep" + fetch.suggested_frameworks: true +} + +languageCode { + assamese : "as" + bengali : "bn" + english : "en" + gujarati : "gu" + hindi : "hi" + kannada : "ka" + marathi : "mr" + odia : "or" + tamil : "ta" + telugu : "te" +} +composite { + search { + url : "{{ proto }}://{{ proxy_server_name }}/action/composite/v3/search" + } +} +cloud_storage_type: "azure" +azure_storage_key: "{{ sunbird_public_storage_account_name }}" +azure_storage_secret: "{{ sunbird_public_storage_account_key }}" +azure_storage_container: "{{ sunbird_content_azure_storage_container }}" + +kafka { + urls : "{{ kafka_urls }}" + topic.send.enable : true + topics.instruction : "{{ env_name }}.learning.job.request" +} + +# DIAL Link Config +dial_service { + api { + base_url : "{{ sunbird_dial_repo_api_base_url }}" + auth_key : "{{ sunbird_dial_repo_api_key }}" + } +} +content.link_dialcode.validation=true +content.link_dialcode.max_limit=10 + +# Content Import API Config +content.import.request_size_limit=1000 +content.import.required_props={{ content_import_required_props }} +content.import.topic_name="{{ env_name }}.auto.creation.job.request" +content.import.remove_props={{ content_import_remove_props }} + +contentTypeToPrimaryCategory { + ClassroomTeachingVideo: "Explanation Content" + ConceptMap: "Learning Resource" + Course: "Course" + CuriosityQuestionSet: "Practice Question Set" + eTextBook: "eTextbook" + ExperientialResource: "Learning Resource" + ExplanationResource: "Explanation Content" + ExplanationVideo: "Explanation Content" + FocusSpot: "Teacher Resource" + LearningOutcomeDefinition: "Teacher Resource" + MarkingSchemeRubric: "Teacher Resource" + PedagogyFlow: "Teacher Resource" + PracticeQuestionSet: "Practice Question Set" + PracticeResource: "Practice Question Set" + SelfAssess: "Course Assessment" + TeachingMethod: "Teacher Resource" + TextBook: "Digital Textbook" + Collection: "Content Playlist" + ExplanationReadingMaterial: "Learning Resource" + LearningActivity: "Learning Resource" + LessonPlan: "Content Playlist" + LessonPlanResource: "Teacher Resource" + PreviousBoardExamPapers: "Learning Resource" + TVLesson: "Explanation Content" + OnboardingResource: "Learning Resource" + ReadingMaterial: "Learning Resource" + Template: "Template" + Asset: "Asset" + Plugin: "Plugin" + LessonPlanUnit: "Lesson Plan Unit" + CourseUnit: "Course Unit" + TextBookUnit: "Textbook Unit" + Asset: "Certificate Template" +} + +resourceTypeToPrimaryCategory { + Learn: "Learning Resource" + Read: "Learning Resource" + Practice: "Learning Resource" + Teach: "Teacher Resource" + Test: "Learning Resource" + Experiment: "Learning Resource" + LessonPlan: "Teacher Resource" +} + +mimeTypeToPrimaryCategory { + "application/vnd.ekstep.h5p-archive": ["Learning Resource"] + "application/vnd.ekstep.html-archive": ["Learning Resource"] + "application/vnd.android.package-archive": ["Learning Resource"] + "video/webm": ["Explanation Content"] + "video/x-youtube": ["Explanation Content"] + "video/mp4": ["Explanation Content"] + "application/pdf": ["Learning Resource", "Teacher Resource"] + "application/epub": ["Learning Resource", "Teacher Resource"] + "application/vnd.ekstep.ecml-archive": ["Learning Resource", "Teacher Resource"] + "text/x-url": ["Learnin Resource", "Teacher Resource"] +} + +objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" + +#Default objectCategory mapping for channel +channel { + content{ + primarycategories=["Course Assessment", "eTextbook", "Explanation Content", "Learning Resource", "Practice Question Set", "Teacher Resource"] + additionalcategories=["Classroom Teaching Video", "Concept Map", "Curiosity Question Set", "Experiential Resource", "Explanation Video", "Focus Spot", "Learning Outcome Definition", "Lesson Plan", "Marking Scheme Rubric", "Pedagogy Flow", "Previous Board Exam Papers", "TV Lesson", "Textbook"] + } + collection { + primarycategories=["Content Playlist", "Course", "Digital Textbook"] + additionalcategories=["Textbook", "Lesson Plan"] + } + asset { + primarycategories=["Asset", "CertAsset", "Certificate Template"] + additionalcategories=[] + } +} \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/content-service_logback.xml b/ansible/roles/stack-sunbird/templates/content-service_logback.xml new file mode 100644 index 000000000..1d3fdd0c6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/content-service_logback.xml @@ -0,0 +1,60 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + + + + + + + + + + %msg + + + {{env_name}}.telemetry.raw + + + + + + + + + bootstrap.servers={{lp_kafka_url}} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/learner-service_logback.xml b/ansible/roles/stack-sunbird/templates/learner-service_logback.xml new file mode 100644 index 000000000..b0509c2dc --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/learner-service_logback.xml @@ -0,0 +1,117 @@ + + + + + + + + + + + + timestamp + msg + lname + tname + [ignore] + [ignore] + exception + + + 30 + 2048 + sun\.reflect\..*\.invoke.* + true + true + + + + + + + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSX + Etc/UTC + + timestamp + msg + lname + tname + [ignore] + [ignore] + + + + + + + + + + timestamp + msg + lname + tname + [ignore] + [ignore] + exception + + + 30 + 2048 + sun\.reflect\..*\.invoke.* + true + true + + + + + + + + %msg + + + ${ENV_NAME}.telemetry.raw + + + + + + + + + bootstrap.servers=${SUNBIRD_KAFKA_URL} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/lms-service_logback.xml b/ansible/roles/stack-sunbird/templates/lms-service_logback.xml new file mode 100644 index 000000000..c6e8485a6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/lms-service_logback.xml @@ -0,0 +1,119 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + + + + + + + + + + %msg + + + ${sunbird_env_name}.telemetry.raw + + + + + + + + + bootstrap.servers=${kafka_urls} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + %msg + + + ${sunbird_env_name}.db.query.events + + + + + + + + + bootstrap.servers=${kafka_urls} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + timestamp + msg + lname + tname + [ignore] + [ignore] + exception + + + 30 + 2048 + sun\.reflect\..*\.invoke.* + true + true + + + + + + + + + + + + + + + + + + diff --git a/ansible/roles/stack-sunbird/templates/search-service_application.conf b/ansible/roles/stack-sunbird/templates/search-service_application.conf new file mode 100644 index 000000000..2e8783892 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/search-service_application.conf @@ -0,0 +1,349 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ +# Play uses HOCON as its configuration file format. HOCON has a number +# of advantages over other config formats, but there are two things that +# can be used when modifying settings. +# +# You can include other configuration files in this main application.conf file: +#include "extra-config.conf" +# +# You can declare variables and substitute for them: +#mykey = ${some.value} +# +# And if an environment variable exists when there is no other substitution, then +# HOCON will fall back to substituting environment variable: +#mykey = ${JAVA_HOME} + +## Akka +# https://www.playframework.com/documentation/latest/ScalaAkka#Configuration +# https://www.playframework.com/documentation/latest/JavaAkka#Configuration +# ~~~~~ +# Play uses Akka internally and exposes Akka Streams and actors in Websockets and +# other streaming HTTP responses. +akka { + # "akka.log-config-on-start" is extraordinarly useful because it log the complete + # configuration at INFO level, including defaults and overrides, so it s worth + # putting at the very top. + # + # Put the following in your conf/logback.xml file: + # + # + # + # And then uncomment this line to debug the configuration. + # + #log-config-on-start = true + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + actors-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + actor { + deployment { + /searchActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + /healthActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + } + } +} + +## Secret key +# http://www.playframework.com/documentation/latest/ApplicationSecret +# ~~~~~ +# The secret key is used to sign Play's session cookie. +# This must be changed for production, but we don't recommend you change it in this file. +play.http.secret.key = a-long-secret-to-calm-the-rage-of-the-entropy-gods + +## Modules +# https://www.playframework.com/documentation/latest/Modules +# ~~~~~ +# Control which modules are loaded when Play starts. Note that modules are +# the replacement for "GlobalSettings", which are deprecated in 2.5.x. +# Please see https://www.playframework.com/documentation/latest/GlobalSettings +# for more information. +# +# You can also extend Play functionality by using one of the publically available +# Play modules: https://playframework.com/documentation/latest/ModuleDirectory +play.modules { + # By default, Play will load any class called Module that is defined + # in the root package (the "app" directory), or you can define them + # explicitly below. + # If there are any built-in modules that you want to enable, you can list them here. + #enabled += my.application.Module + + # If there are any built-in modules that you want to disable, you can list them here. + #disabled += "" + enabled += modules.SearchModule +} + +## IDE +# https://www.playframework.com/documentation/latest/IDE +# ~~~~~ +# Depending on your IDE, you can add a hyperlink for errors that will jump you +# directly to the code location in the IDE in dev mode. The following line makes +# use of the IntelliJ IDEA REST interface: +#play.editor="http://localhost:63342/api/file/?file=%s&line=%s" + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# https://www.playframework.com/documentation/latest/ScalaI18N +# ~~~~~ +# Play comes with its own i18n settings, which allow the user's preferred language +# to map through to internal messages, or allow the language to be stored in a cookie. +play.i18n { + # The application languages + langs = [ "en" ] + + # Whether the language cookie should be secure or not + #langCookieSecure = true + + # Whether the HTTP only attribute of the cookie should be set to true + #langCookieHttpOnly = true +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + + ## Action Creator + # https://www.playframework.com/documentation/latest/JavaActionCreator + # ~~~~~ + #actionCreator = null + + ## ErrorHandler + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # If null, will attempt to load a class called ErrorHandler in the root package, + #errorHandler = null + + ## Session & Flash + # https://www.playframework.com/documentation/latest/JavaSessionFlash + # https://www.playframework.com/documentation/latest/ScalaSessionFlash + # ~~~~~ + session { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + + # Sets the max-age field of the cookie to 5 minutes. + # NOTE: this only sets when the browser will discard the cookie. Play will consider any + # cookie value with a valid signature to be a valid session forever. To implement a server side session timeout, + # you need to put a timestamp in the session and check it at regular intervals to possibly expire it. + #maxAge = 300 + + # Sets the domain on the session cookie. + #domain = "example.com" + } + + flash { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + } +} + +play.http.parser.maxDiskBuffer = 10MB +parsers.anyContent.maxLength = 10MB + +play.server.provider = play.core.server.NettyServerProvider + +## Netty Provider +# https://www.playframework.com/documentation/latest/SettingsNetty +# ~~~~~ +play.server.netty { + # Whether the Netty wire should be logged + log.wire = true + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + transport = "native" +} + +## WS (HTTP Client) +# https://www.playframework.com/documentation/latest/ScalaWS#Configuring-WS +# ~~~~~ +# The HTTP client primarily used for REST APIs. The default client can be +# configured directly, but you can also create different client instances +# with customized settings. You must enable this by adding to build.sbt: +# +# libraryDependencies += ws // or javaWs if using java +# +play.ws { + # Sets HTTP requests not to follow 302 requests + #followRedirects = false + + # Sets the maximum number of open HTTP connections for the client. + #ahc.maxConnectionsTotal = 50 + + ## WS SSL + # https://www.playframework.com/documentation/latest/WsSSL + # ~~~~~ + ssl { + # Configuring HTTPS with Play WS does not require programming. You can + # set up both trustManager and keyManager for mutual authentication, and + # turn on JSSE debugging in development with a reload. + #debug.handshake = true + #trustManager = { + # stores = [ + # { type = "JKS", path = "exampletrust.jks" } + # ] + #} + } +} + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +# libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +## Filter Configuration +# https://www.playframework.com/documentation/latest/Filters +# ~~~~~ +# There are a number of built-in filters that can be enabled and configured +# to give Play greater security. +# +play.filters { + + # Enabled filters are run automatically against Play. + # CSRFFilter, AllowedHostFilters, and SecurityHeadersFilters are enabled by default. + enabled = [filters.AccessLogFilter] + + # Disabled filters remove elements from the enabled list. + # disabled += filters.CSRFFilter + + + ## CORS filter configuration + # https://www.playframework.com/documentation/latest/CorsFilter + # ~~~~~ + # CORS is a protocol that allows web applications to make requests from the browser + # across different domains. + # NOTE: You MUST apply the CORS configuration before the CSRF filter, as CSRF has + # dependencies on CORS settings. + cors { + # Filter paths by a whitelist of path prefixes + #pathPrefixes = ["/some/path", ...] + + # The allowed origins. If null, all origins are allowed. + #allowedOrigins = ["http://www.example.com"] + + # The allowed HTTP methods. If null, all methods are allowed + #allowedHttpMethods = ["GET", "POST"] + } + + ## Security headers filter configuration + # https://www.playframework.com/documentation/latest/SecurityHeaders + # ~~~~~ + # Defines security headers that prevent XSS attacks. + # If enabled, then all options are set to the below configuration by default: + headers { + # The X-Frame-Options header. If null, the header is not set. + #frameOptions = "DENY" + + # The X-XSS-Protection header. If null, the header is not set. + #xssProtection = "1; mode=block" + + # The X-Content-Type-Options header. If null, the header is not set. + #contentTypeOptions = "nosniff" + + # The X-Permitted-Cross-Domain-Policies header. If null, the header is not set. + #permittedCrossDomainPolicies = "master-only" + + # The Content-Security-Policy header. If null, the header is not set. + #contentSecurityPolicy = "default-src 'self'" + } + + ## Allowed hosts filter configuration + # https://www.playframework.com/documentation/latest/AllowedHostsFilter + # ~~~~~ + # Play provides a filter that lets you configure which hosts can access your application. + # This is useful to prevent cache poisoning attacks. + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + #allowed = [".example.com", "localhost:9000"] + } +} + +play.http.parser.maxMemoryBuffer = 50MB +akka.http.parsing.max-content-length = 50MB + +schema.base_path="{{kp_schema_base_path | default('/home/sunbird/search-service-1.0-SNAPSHOT/schemas')}}" + + +telemetry_env="{{ sunbird_instance }}" +installation.id="{{sunbird_installation}}" + +# ElasticSearch Configuration +ekstepPlatformApiUserId="search-service" +search.es_conn_info="{{ search_index_host }}" +search.fields.query=["name^100","title^100","lemma^100","code^100","domain","subject","description^10","keywords^100","ageGroup^10","filter^10","theme^10","genre^10","objects^25","contentType^100","language^200","teachingMode^25","skills^10","learningObjective^10","curriculum^100","gradeLevel^100","developer^100","attributions^10","identifier^100","IL_UNIQUE_ID^100","owner^50","board^100","relatedBoards^100","creator^100", "dialcodes^100","text","words","releaseNotes"] +search.fields.date=["lastUpdatedOn","createdOn","versionDate","lastSubmittedOn","lastPublishedOn"] +search.fields.mode_collection=["identifier","name","objectType","contentType","mimeType","size","childNodes","board","subject","medium","gradeLevel","appIcon","resourceType","origin","originData"] +search.batch.size=500 +search.connection.timeout=30 + +language.map={"Hindi":"hi", "English":"en", "Telugu":"te", "Kannada":"ka", "Tamil":"ta", "Assamese":"as", "Bengali":"bn", "Bodo":"bo", "Gujarati":"gu", "Konkani":"ko", "Malayalam":"ml", "Marathi":"mr", "Nepali":"ne", "Odia":"or", "Punjabi":"pj", "Sanskrit":"san"} + +# Configuration for default channel ID +channel.default="in.ekstep" +compositesearch.index.name="{{ compositesearch_index_name }}" + +content.tagging.backward_enable=false +content.tagging.property=["subject","medium"] +search.payload.log_enable=true diff --git a/ansible/roles/stack-sunbird/templates/search-service_logback.xml b/ansible/roles/stack-sunbird/templates/search-service_logback.xml new file mode 100644 index 000000000..1d3fdd0c6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/search-service_logback.xml @@ -0,0 +1,60 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + + + + + + + + + + %msg + + + {{env_name}}.telemetry.raw + + + + + + + + + bootstrap.servers={{lp_kafka_url}} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/stack_content_service.yml b/ansible/roles/stack-sunbird/templates/stack_content_service.yml deleted file mode 100644 index dff2a9c34..000000000 --- a/ansible/roles/stack-sunbird/templates/stack_content_service.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: '3.1' - -services: - content_service: - image: "{{hub_org}}/{{image_name}}:{{image_tag}}" - deploy: - replicas: {{ content_replicas | default(1) }} - resources: - reservations: - memory: "{{ content_reservation_memory | default('64M') }}" - limits: - memory: "{{ content_limit_memory | default('256M') }}" - healthcheck: - test: curl -f http://localhost:5000/health || exit 1 - interval: 10s - timeout: 5s - retries: 5 - env_file: - /home/deployer/env/sunbird_content_service.env - ports: - - "5000:5000" - networks: - - application_default - -networks: - application_default: - external: true diff --git a/ansible/roles/stack-sunbird/templates/stack_player.yml b/ansible/roles/stack-sunbird/templates/stack_player.yml index caf414f12..02722c0e4 100644 --- a/ansible/roles/stack-sunbird/templates/stack_player.yml +++ b/ansible/roles/stack-sunbird/templates/stack_player.yml @@ -4,12 +4,14 @@ services: player: image: "{{hub_org}}/{{image_name}}:{{image_tag}}" deploy: - replicas: {{ player_replicas | default(1) }} + replicas: {{ player_replicas }} resources: reservations: - memory: "{{ player_reservation_memory | default('64M') }}" + memory: "{{ player_reservation_memory }}" + cpus: "{{ player_reserve_cpu }}" limits: - memory: "{{ player_limit_memory | default('256M') }}" + memory: "{{ player_limit_memory }}" + cpus: "{{ player_limit_cpu }}" volumes: - "{% if player_tenant_dir is defined and player_tenant_dir %}{{ player_tenant_dir }}:/home/sunbird/dist/tenant{% else %}/home/sunbird/dist/tenant{% endif %}" env_file: diff --git a/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env b/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env index 4f8512c70..cba2fdd44 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_cert-registry-service.env @@ -1,3 +1,14 @@ cert_service_base_url=http://cert-service:9000 sunbird_es_host={{sunbird_es_host}} sunbird_es_port={{sunbird_es_port}} +sunbird_cassandra_keyspace={{sunbird_cassandra_keyspace|d('sunbird')}} +sunbird_cassandra_username={{sunbird_cassandra_username|d('cassandra')}} +sunbird_cassandra_password={{sunbird_cassandra_password|d('password')}} +sunbird_cassandra_consistency_level=quorum +{% if groups['cassandra-2'] is defined %} +sunbird_cassandra_host={{groups['cassandra']|join(',')}} +sunbird_cassandra_port=9042 +{% else %} +sunbird_cassandra_host={{sunbird_cassandra_host}} +sunbird_cassandra_port=9042 +{% endif %} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env index 8bbafae36..686df40e6 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_cert-service.env @@ -1,8 +1,13 @@ CONTAINER_NAME={{cert_service_container_name}} CLOUD_STORAGE_TYPE={{cert_service_cloud_storage_type}} -AZURE_STORAGE_SECRET={{sunbird_azure_storage_key}} -AZURE_STORAGE_KEY={{sunbird_azure_storage_account}} +AZURE_STORAGE_SECRET={{sunbird_private_storage_account_key}} +AZURE_STORAGE_KEY={{sunbird_private_storage_account_name}} sunbird_cert_domain_url={{proto}}://{{proxy_server_name}} sunbird_cert_enc_service_url=http://enc-service:8013 download_link_expiry_timeout=600 -es_conn_info={{sunbird_es_host}}:{{sunbird_es_http_port}} \ No newline at end of file +es_conn_info={{groups['es']|join(':9200,')}}:9200 +ITEXT_LICENSE_ENABLED={{itext_license_enabled}} +ITEXT_LICENSE_PATH=/home/sunbird/itext_trail_license.xml +PUBLIC_AZURE_STORAGE_KEY={{sunbird_public_storage_account_name}} +PUBLIC_AZURE_STORAGE_SECRET={{sunbird_public_storage_account_key}} +PUBLIC_CONTAINER_NAME={{sunbird_cert_qr_container_name}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_config-service.env b/ansible/roles/stack-sunbird/templates/sunbird_config-service.env index 25ac319ae..75fe5510b 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_config-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_config-service.env @@ -1,7 +1,7 @@ bind_hostname=0.0.0.0 cloud_storage_type={{cloud_storage_type}} -azure_storage_key={{azure_storage_key}} -azure_storage_secret={{azure_storage_secret}} +azure_storage_key={{sunbird_public_storage_account_name}} +azure_storage_secret={{sunbird_public_storage_account_key}} azure_storage_container={{azure_storage_container}} aws_storage_key={{aws_storage_key}} aws_storage_secret={{aws_storage_secret}} diff --git a/tests/ci/build_info b/ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env similarity index 100% rename from tests/ci/build_info rename to ansible/roles/stack-sunbird/templates/sunbird_discussions-mw-service.env diff --git a/ansible/roles/stack-sunbird/templates/sunbird_druid-proxy-api.env b/ansible/roles/stack-sunbird/templates/sunbird_druid-proxy-api.env new file mode 100644 index 000000000..704bb4724 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_druid-proxy-api.env @@ -0,0 +1,8 @@ +# Configuration for Druid + +druid_proxy_api_endPoint=/druid/v2/ +druid_proxy_api_port=8082 +druid_host={{sunbird_druid_broker_host}} +druid_port=8082 + + diff --git a/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env b/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env new file mode 100644 index 000000000..32438e543 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_groups-service.env @@ -0,0 +1,36 @@ +{% if groups['cassandra-2'] is defined %} +sunbird_cassandra_host={{groups['cassandra']|join(',')}} +sunbird_cassandra_port=9042,9042,9042 +sunbird_cassandra_consistency_level=quorum +{% else %} +sunbird_cassandra_host={{sunbird_cassandra_host}} +sunbird_cassandra_port=9042 +sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}} +{% endif %} +sunbird_cassandra_password={{sunbird_cassandra_password|d('password')}} +sunbird_cassandra_username={{sunbird_cassandra_username|d('cassandra')}} +LEARNER_SERVICE_PORT=http://learner-service:9000 +sunbird_redis_host={{sunbird_redis_host}} +sunbird_redis_port={{sunbird_redis_port|default(6379)}} +CONTENT_SERVICE_PORT=http://search-service:9000 +sunbird_sso_publickey={{sunbird_sso_publickey}} +sunbird_sso_url={{keycloak_auth_server_url}}/ +sunbird_sso_realm={{keycloak_realm}} +sunbird_sso_client_id={{sunbird_sso_client_id}} +sunbird_sso_client_secret={{sunbird_sso_client_secret | d('')}} +sunbird_sso_username={{sunbird_sso_username}} +sunbird_sso_password={{sunbird_sso_password}} +sunbird_keycloak_user_federation_provider_id={{core_vault_sunbird_keycloak_user_federation_provider_id}} +sunbird_keycloak_required_action_link_expiration_seconds={{sunbird_keycloak_required_action_link_expiration_seconds}} +accesstoken.publickey.basepath={{learner_device_basepath | default('/keys/')}} +sunbird_user_service_search_url=/private/user/v1/search +sunbird_cs_search_url=/v3/search +ENV_NAME={{env_name}} +SUNBIRD_KAFKA_URL={{kafka_urls}} +max_group_members_limit={{max_group_members_limit | default(150)}} +max_activity_limit={{max_activity_limit | default(100)}} +max_group_limit={{max_group_limit | default(50)}} +enable_userid_redis_cache={{enable_userid_redis_cache | default(true)}} +groups_redis_ttl={{groups_redis_ttl | default(86400)}} +user_redis_ttl={{user_redis_ttl | default(3600)}} + diff --git a/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env b/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env index 825d56a6f..ec8132a2a 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_knowledge-mw-service.env @@ -1,15 +1,15 @@ sunbird_content_plugin_base_url={{sunbird_ekstep_proxy_base_url}} sunbird_learner_service_api_key={{sunbird_api_auth_token}} sunbird_learner_service_base_url={{sunbird_learner_service_base_url}} -sunbird_keycloak_auth_server_url={{keycloak_auth_server_url}}/ +sunbird_keycloak_auth_server_url={{keycloak_auth_server_url}} sunbird_keycloak_realm={{keycloak_realm}} sunbird_keycloak_client_id={{sunbird_keycloak_client_id}} sunbird_keycloak_public={{sunbird_keycloak_public}} sunbird_cache_store={{sunbird_cache_store}} sunbird_cache_ttl={{sunbird_cache_ttl}} sunbird_image_storage_url={{sunbird_image_storage_url}} -sunbird_azure_account_name={{sunbird_account_name}} -sunbird_azure_account_key={{sunbird_account_key}} +sunbird_azure_account_name={{sunbird_public_storage_account_name}} +sunbird_azure_account_key={{sunbird_public_storage_account_key}} sunbird_dial_code_registry_url=https://{{proxy_server_name}}/dial/ sunbird_telemetry_sync_batch_size={{sunbird_telemetry_sync_batch_size}} sunbird_environment={{sunbird_environment}} @@ -19,11 +19,11 @@ sunbird_content_service_blacklisted_channels={{content_service_blacklisted_chann sunbird_default_channel={{sunbird_default_channel}} sunbird_content_repo_api_base_url={{sunbird_content_repo_api_base_url}} sunbird_content_repo_api_key={{sunbird_content_repo_api_key}} -sunbird_search_service_api_base_url={{sunbird_search_service_api_base_url}} +sunbird_search_service_api_base_url={{knowledge_mw_sunbird_search_service_api_base_url | default('http://search-service:9000')}} sunbird_search_service_api_key={{sunbird_search_service_api_key}} sunbird_dial_repo_api_base_url={{sunbird_dial_repo_api_base_url}} sunbird_dial_repo_api_key={{sunbird_dial_repo_api_key}} -sunbird_plugin_repo_api_base_url={{sunbird_plugin_repo_api_base_url}} +sunbird_plugin_repo_api_base_url={{knowledge_mw_sunbird_plugin_repo_api_base_url | default('http://search-service:9000')}} sunbird_plugin_repo_api_key={{sunbird_plugin_repo_api_key}} sunbird_data_service_api_base_url={{sunbird_data_service_api_base_url}} sunbird_data_service_api_key={{sunbird_data_service_api_key}} @@ -56,3 +56,5 @@ sunbird_cassandra_consistency_level={{sunbird_cassandra_consistency_level}} {% endif %} # Release-1.15 sunbird_health_check_enable={{sunbird_health_check_enable}} +# Release-3.0.0 +sunbird_keycloak_public_key={{core_vault_sunbird_sso_publickey}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env index f24d32f01..a03b1c5b4 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_learner-service.env @@ -4,8 +4,9 @@ sunbird_sso_realm={{keycloak_realm}} sunbird_sso_username={{sunbird_sso_username}} sunbird_sso_password={{sunbird_sso_password}} sunbird_sso_client_id={{sunbird_sso_client_id}} -sunbird_es_host={{sunbird_es_host}} -sunbird_es_port={{sunbird_es_port}} +sunbird_sso_client_secret={{sunbird_sso_client_secret | d('')}} +sunbird_es_host={{groups['es']|join(',')}} +sunbird_es_port=9300,9300,9300 sunbird_cassandra_username=cassandra sunbird_cassandra_password=password actor_hostname=actor-service @@ -27,8 +28,8 @@ sunbird_mail_server_password={{sunbird_mail_server_password}} sunbird_mail_server_from_email={{sunbird_mail_server_from_email}} sunbird_encryption_key={{sunbird_encryption_key}} sunbird_encryption_mode={{sunbird_encryption_mode}} -sunbird_account_name={{sunbird_account_name}} -sunbird_account_key={{sunbird_account_key}} +sunbird_account_name={{sunbird_public_storage_account_name}} +sunbird_account_key={{sunbird_public_storage_account_key}} sunbird_quartz_mode={{sunbird_sunbird_quartz_mode}} sunbird_env_logo_url={{sunbird_env_logo_url}} sunbird_web_url={{sunbird_web_url}} @@ -62,7 +63,7 @@ sunbird_telemetry_base_url=http://telemetry-service:9001 telemetry_queue_threshold_value=100 sunbird_default_channel={{sunbird_default_channel}} sunbird_api_mgr_base_url=http://knowledge-mw-service:5000 -sunbird_cs_base_url={{sunbird_cs_base_url}} +sunbird_cs_base_url=http://knowledge-mw-service:5000 sunbird_cs_search_path=/v1/content/search sunbird_user_bulk_upload_size={{sunbird_user_bulk_upload_size}} @@ -91,8 +92,8 @@ sunbird_keycloak_user_federation_provider_id={{core_vault_sunbird_keycloak_user_ sunbird_gzip_enable={{sunbird_gzip_enable}} sunbird_course_metrics_base_url={{sunbird_course_metrics_base_url}} sunbird_gzip_size_threshold={{sunbird_gzip_size_threshold | default(262144)}} -sunbird_analytics_blob_account_name={{sunbird_analytics_blob_account_name}} -sunbird_analytics_blob_account_key={{sunbird_analytics_blob_account_key}} +sunbird_analytics_blob_account_name={{sunbird_private_storage_account_name}} +sunbird_analytics_blob_account_key={{sunbird_private_storage_account_key}} # Optional for caching sunbird_cache_enable={{sunbird_cache_enable | default(false)}} # Set below variables if above true @@ -107,3 +108,16 @@ sunbird_subdomain_keycloak_base_url={{sunbird_subdomain_keycloak_base_url | d('' #Required by Learner-service to call cert service sunbird_cert_service_base_url=http://cert-service:9000 quartz_shadow_user_migration_timer={{sunbird_quartz_shadow_user_migration_timer}} +sunbird_otp_allowed_attempt={{sunbird_otp_allowed_attempt|default(2)}} +SUNBIRD_KAFKA_URL={{kafka_urls}} +ENV_NAME={{env_name}} +notification_service_base_url=http://notification-service:9000 +# Release-3.4.0 +feed_limit={{feed_limit | default(30)}} + +## Access Keys ## +accesstoken.publickey.basepath={{learner_device_basepath | default('/keys/')}} + +## Captcha keys ## +google_captcha_private_key={{google_captcha_private_key | default("")}} +google_captcha_mobile_private_key={{google_captcha_mobile_private_key | default("")}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env b/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env deleted file mode 100644 index 8db5e1e1c..000000000 --- a/ansible/roles/stack-sunbird/templates/sunbird_learning-service.env +++ /dev/null @@ -1,48 +0,0 @@ -# Cassandra Configuration -cassandra.lp.connection={{ lp_cassandra_connection }} -cassandra.lpa.connection={{ dp_cassandra_connection }} - -# Redis Configuration -redis.host=localhost -redis.port=6379 -redis.maxConnections=128 - -#--Maximum Content Package File Size Limit in Bytes (50 MB) -MAX_CONTENT_PACKAGE_FILE_SIZE_LIMIT=52428800 - -#--Maximum Asset File Size Limit in Bytes (20 MB) -MAX_ASSET_FILE_SIZE_LIMIT=20971520 - -#--No of Retry While File Download Fails -RETRY_ASSET_DOWNLOAD_COUNT=1 - -#Current environment -cloud_storage.env=dev - -# Configuration -graph.dir=/data/graphDB -akka.request_timeout=30 -environment.id={{ environment_id }} -graph.ids=["domain"] -graph.passport.key.base={{ graph_passport_key }} -route.domain={{ learning_neo4j_bolt_url }} -route.bolt.write.domain={{ learning_write_elb_url }} -route.bolt.read.domain={{ learning_read_elb_url }} -route.all={{ language_neo4j_bolt_url }} -route.bolt.write.all={{ language_write_elb_url }} -route.bolt.read.all={{ language_read_elb_url }} - -shard.id={{ mw_shard_id }} -platform.auth.check.enabled=false -platform.cache.ttl=3600000 - -#Top N Config for Search Telemetry -telemetry_env=dev - -installation.id=ekstep - -channel.default=in.ekstep - -content.cache.ttl=86400 -content.cache.read=true -content.cache.hierarchy=true \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env index 67652c19d..389b8a412 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_lms-service.env @@ -4,6 +4,7 @@ sunbird_sso_realm={{keycloak_realm}} sunbird_sso_username={{sunbird_sso_username}} sunbird_sso_password={{sunbird_sso_password}} sunbird_sso_client_id={{sunbird_sso_client_id}} +sunbird_sso_client_secret={{sunbird_sso_client_secret | d('')}} sunbird_es_host={{sunbird_es_host}} sunbird_es_port={{sunbird_es_port}} sunbird_cassandra_username=cassandra @@ -18,8 +19,9 @@ sunbird_pg_user={{sunbird_pg_user}} sunbird_pg_password={{sunbird_pg_password}} sunbird_installation={{sunbird_installation}} sunbird_analytics_api_base_url={{sunbird_analytics_api_base_url}} -sunbird_search_service_api_base_url={{sunbird_search_service_api_base_url}} -ekstep_api_base_url={{sunbird_content_repo_api_base_url}} +sunbird_search_service_api_base_url={{ lms_search_service_api_base_url | default('http://search-service:9000') }} +ekstep_api_base_url={{ lms_ekstep_api_base_url | default('http://content-service:9000') }} +sunbird_group_service_api_base_url={{sunbird_group_service_api_base_url}} sunbird_mail_server_host={{sunbird_mail_server_host}} sunbird_mail_server_port={{sunbird_mail_server_port}} sunbird_mail_server_username={{sunbird_mail_server_username}} @@ -27,10 +29,9 @@ sunbird_mail_server_password={{sunbird_mail_server_password}} sunbird_mail_server_from_email={{sunbird_mail_server_from_email}} sunbird_encryption_key={{sunbird_encryption_key}} sunbird_encryption_mode={{sunbird_encryption_mode}} -sunbird_account_name={{sunbird_account_name}} -sunbird_account_key={{sunbird_account_key}} +sunbird_account_name={{sunbird_public_storage_account_name}} +sunbird_account_key={{sunbird_public_storage_account_key}} sunbird_quartz_mode={{sunbird_sunbird_quartz_mode}} -sunbird_env_logo_url={{sunbird_env_logo_url}} sunbird_web_url={{sunbird_web_url}} sunbird_fcm_account_key={{sunbird_fcm_account_key}} sunbird_msg_91_auth={{sunbird_msg_91_auth}} @@ -62,7 +63,7 @@ sunbird_telemetry_base_url=http://telemetry-service:9001 telemetry_queue_threshold_value=100 sunbird_default_channel={{sunbird_default_channel}} sunbird_api_mgr_base_url=http://knowledge-mw-service:5000 -sunbird_cs_base_url={{sunbird_cs_base_url}} +sunbird_cs_base_url=http://knowledge-mw-service:5000 sunbird_cs_search_path=/v1/content/search sunbird_env_logo_url={{sunbird_env_logo_url}} @@ -92,18 +93,54 @@ sunbird_keycloak_user_federation_provider_id={{core_vault_sunbird_keycloak_user_ sunbird_gzip_enable={{sunbird_gzip_enable}} sunbird_course_metrics_base_url={{sunbird_course_metrics_base_url}} sunbird_gzip_size_threshold={{sunbird_gzip_size_threshold | default(262144)}} -sunbird_analytics_blob_account_name={{sunbird_analytics_blob_account_name}} -sunbird_analytics_blob_account_key={{sunbird_analytics_blob_account_key}} +sunbird_analytics_blob_account_name={{sunbird_private_storage_account_name}} +sunbird_analytics_blob_account_key={{sunbird_private_storage_account_key}} # Optional for caching sunbird_cache_enable={{sunbird_cache_enable | default(false)}} # Set below variables if above true sunbird_redis_host={{sunbird_redis_host}} sunbird_redis_port={{sunbird_redis_port|default(6379)}} -sunbird_user_org_api_base_url={{sunbird_user_org_api_base_url}} +sunbird_user_org_api_base_url=http://learner-service:9000 kafka_topics_instruction={{kafka_topics_instruction}} kafka_urls={{kafka_urls}} kafka_topics_certificate_instruction={{kafka_topics_certificate_instruction}} kafka_assessment_topic={{kafka_assessment_topic}} +kafka_topics_contentstate_invalid={{kafka_topics_contentstate_invalid}} # Required by lms service to call cert service -sunbird_cert_service_base_url=http://cert-service:9000 \ No newline at end of file +sunbird_cert_service_base_url=http://cert-service:9000 + +learning_service_base_url={{sunbird_content_repo_api_base_url}} +sunbird_user_search_cretordetails_fields=id,firstName,lastName +sunbird_user_service_api_base_url={{sunbird_user_service_api_base_url}} + +limit_managed_user_creation={{limit_managed_user_creation | default(true)}} +managed_user_limit={{managed_user_limit | default(30)}} +sunbird_user_qrcode_courses_limit={{sunbird_user_qrcode_courses_limit|default(5000)}} + +## Access Keys ## +accesstoken.publickey.basepath={{lms_device_basepath | default('/keys/')}} + +learning.content.props.to.add={{ learning_content_props_to_add | default('mimeType,contentType,name,code,description,keywords,framework,copyright,topic')}} +group_activity_agg_cache_ttl={{ group_activity_agg_cache_ttl }} +group_activity_agg_cache_enable={{ group_activity_agg_cache_enable }} +sunbird_env_name={{env_name}} +content.default.fields=contentid,userid,batchid,courseid,completedcount,completionpercentage,lastcompletedtime,status,viewcount + +## Redis connection detail ## +redis.connection.max={{ lms_redis_connection_max | default(64) }} +redis.connection.idle.max={{ lms_redis_connection_idle_max | default(32) }} +redis.connection.idle.min={{ lms_redis_connection_idle_min | default(1) }} +redis.connection.minEvictableIdleTimeSeconds={{ lms_redis_connection_minEvictableIdleTimeSeconds | default(120) }} +redis.connection.timeBetweenEvictionRunsSeconds={{ lms_redis_connection_timeBetweenEvictionRunsSeconds | default(300) }} +redis.experimentIndex={{ lms_redis_experimentIndex | default(10) }} +redis.dbIndex={{ lms_redis_dbIndex | default(2) }} +user_enrolments_response_cache_enable={{ user_enrolments_response_cache_enable | default(true)}} +user_enrolments_response_cache_ttl={{ user_enrolments_response_cache_ttl | default(300) }} + +##collection-summary-agg-api +druid_proxy_api_host={{groups['raw-broker'][0]}} +druid_proxy_api_port=8082 +druid_proxy_api_endpoint=/druid/v2/ +collection_summary_agg_data_source={{ summary_agg_data_source | default('telemetry-events') }} +collection_summary_agg_cache_ttl={{ summary_agg_ttl | default(21600) }} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_nodebb.env b/ansible/roles/stack-sunbird/templates/sunbird_nodebb.env new file mode 100644 index 000000000..1f5f3155c --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_nodebb.env @@ -0,0 +1,4 @@ +AZURE_STORAGE_ACCOUNT={{sunbird_nodebb_storage_account}} +AZURE_STORAGE_ACCESS_KEY={{sunbird_nodebb_storage_key}} +AZURE_STORAGE_CONTAINER={{sunbird_nodebb_container}} +AZURE_STORAGE_HOSTNAME={{sunbird_nodebb_hostname}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_player.env b/ansible/roles/stack-sunbird/templates/sunbird_player.env index 571a02583..43a9d0cb9 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_player.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_player.env @@ -15,8 +15,8 @@ sunbird_api_auth_token={{sunbird_api_auth_token}} sunbird_enable_permission_check=1 sunbird_cassandra_urls={{sunbird_cassandra_urls}} sunbird_session_store_type={{sunbird_session_store_type}} -AZURE_STORAGE_ACCOUNT={{sunbird_portal_azure_storage_account}} -AZURE_STORAGE_KEY={{sunbird_portal_azure_storage_key}} +AZURE_STORAGE_ACCOUNT={{sunbird_public_storage_account_name}} +AZURE_STORAGE_KEY={{sunbird_public_storage_account_key}} sunbird_dataservice_url={{sunbird_dataservice_url}} sunbird_telemetry_packet_size={{sunbird_telemetry_packet_size}} sunbird_environment={{sunbird_environment}} @@ -43,8 +43,8 @@ sunbird_google_keycloak_client_id={{sunbird_google_keycloak_client_id}} sunbird_google_keycloak_secret={{sunbird_google_keycloak_secret}} sunbird_help_link_visibility={{sunbird_help_link_visibility|default('FALSE')}} sunbird_azure_report_container_name={{sunbird_azure_report_container_name}} -sunbird_azure_account_name={{sunbird_portal_azure_storage_account}} -sunbird_azure_account_key={{sunbird_portal_azure_storage_key}} +sunbird_azure_account_name={{sunbird_private_storage_account_name}} +sunbird_azure_account_key={{sunbird_private_storage_account_key}} sunbird_response_cache_ttl={{sunbird_response_cache_ttl}} sunbird_portal_preview_cdn_url={{sunbird_portal_preview_cdn_url}} @@ -71,11 +71,45 @@ sunbird_google_android_keycloak_secret={{sunbird_google_android_keycloak_secret} sunbird_trampoline_android_keycloak_client_id={{sunbird_trampoline_android_keycloak_client_id}} sunbird_trampoline_android_keycloak_secret={{sunbird_trampoline_android_keycloak_secret}} sunbird_android_keycloak_client_id={{sunbird_android_keycloak_client_id}} - +sunbird_ext_plugin_url=http://player:3000/plugin/ #Release-2.3.5 sunbird_portal_merge_auth_server_url={{sunbird_subdomain_keycloak_base_url | d('')}} #Release-2.5.0 -sunbird_device_api={{sunbird_device_api}} +sunbird_device_api=http://kong:8000/ desktop_app_storage_url={{desktop_app_storage_url | d('')}} sunbird_portal_base_url={{sunbird_portal_base_url | d('')}} + +#Release-2.6.5 +sunbird_portal_updateLoginTimeEnabled={{sunbird_portal_updateLoginTimeEnabled|lower}} + +#Release-2.8.6 +sunbird_portal_slugForProminentFilter={{sunbird_portal_slugForProminentFilter | default("")}} +sunbird_report_service_url={{sunbird_report_service_url}} + +#Release-2.10.0 +google_captcha_private_key={{google_captcha_private_key | default("")}} + +#Release-3.0.0 +sunbird_collectionEditorURL=collection-editor/index.html +sunbird_contentEditorURL=content-editor/index.html +sunbird_genericEditorURL=generic-editor/index.html +sunbird_keycloak_public_key={{core_vault_sunbird_sso_publickey}} +sunbird_bot_configured={{sunbird_bot_configured|default('true')}} +sunbird_bot_service_URL=/chatapi/bot + +#Release-3.1.0 +sunbird_p1_reCaptcha_enabled={{sunbird_p1_reCaptcha_enabled|default('true')}} +sunbird_p2_reCaptcha_enabled={{sunbird_p2_reCaptcha_enabled|default('true')}} +sunbird_p3_reCaptcha_enabled={{sunbird_p3_reCaptcha_enabled|default('true')}} +sunbird_phraseApp_token={{sunbird_phraseApp_token|default('')}} +sunbird_super_admin_slug={{sunbird_super_admin_slug|default('sunbird')}} +sunbird_enable_sso={{sunbird_enable_sso|default('true')}} +sunbird_api_request_timeout={{sunbird_api_request_timeout|default(60000)}} +sunbird_session_ttl={{sunbird_session_ttl|default(86400000)}} + +#Release-3.2.0 +reportsListVersion={{reportsListVersion|default('v1')}} + +#Release-3.2.7 +sunbird_data_product_service=http://kong:8000/ diff --git a/ansible/roles/stack-sunbird/templates/sunbird_print-service.env b/ansible/roles/stack-sunbird/templates/sunbird_print-service.env new file mode 100644 index 000000000..50c203dae --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_print-service.env @@ -0,0 +1,8 @@ +service_threads=2 +sunbird_azure_account_name={{sunbird_public_storage_account_name}} +sunbird_azure_account_key={{sunbird_public_storage_account_key}} +sunbird_azure_container_name={{sunbird_content_azure_storage_container}} + +sunbird_pvt_azure_account_name={{sunbird_private_storage_account_name}} +sunbird_pvt_azure_account_key={{sunbird_private_storage_account_key}} +sunbird_pvt_azure_container_name={{cert_service_container_name}} diff --git a/ansible/roles/stack-sunbird/templates/sunbird_report-service.env b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env new file mode 100644 index 000000000..9d609a0cc --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/sunbird_report-service.env @@ -0,0 +1,12 @@ +SUNBIRD_REPORTS_DB_HOST={{dp_postgres_host}} +SUNBIRD_REPORTS_DB_NAME={{dp_postgres_db}} +SUNBIRD_REPORTS_DB_PASSWORD={{dp_postgres_password}} +SUNBIRD_REPORTS_DB_PORT=5432 +SUNBIRD_REPORTS_DB_USER={{dp_postgres_username}} +SUNBIRD_SERVER_PORT=3030 +SUNBIRD_BASE_REPORT_URL=report +SUNBIRD_REPORTS_TABLE_NAME=report +SUNBIRD_REPORT_SUMMARY_TABLE_NAME={{sunbird_report_summary_table_name}} +SUNBIRD_ENV="https://{{domain_name}}" +DEACTIVATE_JOB_API_HOST=http://kong:8000/data/v1/report/jobs/deactivate +DEACTIVATE_JOB_API_KEY={{sunbird_api_auth_token}} \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/sunbird_telemetry-service.env b/ansible/roles/stack-sunbird/templates/sunbird_telemetry-service.env index 59b03f1cd..1fe41c703 100644 --- a/ansible/roles/stack-sunbird/templates/sunbird_telemetry-service.env +++ b/ansible/roles/stack-sunbird/templates/sunbird_telemetry-service.env @@ -4,6 +4,7 @@ telemetry_local_storage_enabled=true telemetry_local_storage_type={{sunbird_telemetry_dispatchers}} telemetry_kafka_broker_list={{sunbird_telemetry_kafka_servers}} telemetry_kafka_topic={{sunbird_telemetry_kafka_topic}} +telemetry_kafka_compression=snappy # Below is the config to enable Telemetry Proxy. To send telemetry data to ekstep telemetry_proxy_enabled=false diff --git a/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf b/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf new file mode 100644 index 000000000..66a27cfe6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/taxonomy-service_application.conf @@ -0,0 +1,392 @@ +# This is the main configuration file for the application. +# https://www.playframework.com/documentation/latest/ConfigFile +# ~~~~~ +# Play uses HOCON as its configuration file format. HOCON has a number +# of advantages over other config formats, but there are two things that +# can be used when modifying settings. +# +# You can include other configuration files in this main application.conf file: +#include "extra-config.conf" +# +# You can declare variables and substitute for them: +#mykey = ${some.value} +# +# And if an environment variable exists when there is no other substitution, then +# HOCON will fall back to substituting environment variable: +#mykey = ${JAVA_HOME} + +## Akka +# https://www.playframework.com/documentation/latest/ScalaAkka#Configuration +# https://www.playframework.com/documentation/latest/JavaAkka#Configuration +# ~~~~~ +# Play uses Akka internally and exposes Akka Streams and actors in Websockets and +# other streaming HTTP responses. +akka { + # "akka.log-config-on-start" is extraordinarly useful because it log the complete + # configuration at INFO level, including defaults and overrides, so it s worth + # putting at the very top. + # + # Put the following in your conf/logback.xml file: + # + # + # + # And then uncomment this line to debug the configuration. + # + #log-config-on-start = true + default-dispatcher { + # This will be used if you have set "executor = "fork-join-executor"" + fork-join-executor { + # Min number of threads to cap factor-based parallelism number to + parallelism-min = 8 + + # The parallelism factor is used to determine thread pool size using the + # following formula: ceil(available processors * factor). Resulting size + # is then bounded by the parallelism-min and parallelism-max values. + parallelism-factor = 32.0 + + # Max number of threads to cap factor-based parallelism number to + parallelism-max = 64 + + # Setting to "FIFO" to use queue like peeking mode which "poll" or "LIFO" to use stack + # like peeking mode which "pop". + task-peeking-mode = "FIFO" + } + } + actors-dispatcher { + type = "Dispatcher" + executor = "fork-join-executor" + fork-join-executor { + parallelism-min = 8 + parallelism-factor = 32.0 + parallelism-max = 64 + } + # Throughput for default Dispatcher, set to 1 for as fair as possible + throughput = 1 + } + actor { + deployment { + /contentActor + { + router = smallest-mailbox-pool + nr-of-instances = 10 + dispatcher = actors-dispatcher + } + } + } +} + +## Secret key +# http://www.playframework.com/documentation/latest/ApplicationSecret +# ~~~~~ +# The secret key is used to sign Play's session cookie. +# This must be changed for production, but we don't recommend you change it in this file. +play.http.secret.key="{{ graph_passport_key }}" + +## Modules +# https://www.playframework.com/documentation/latest/Modules +# ~~~~~ +# Control which modules are loaded when Play starts. Note that modules are +# the replacement for "GlobalSettings", which are deprecated in 2.5.x. +# Please see https://www.playframework.com/documentation/latest/GlobalSettings +# for more information. +# +# You can also extend Play functionality by using one of the publically available +# Play modules: https://playframework.com/documentation/latest/ModuleDirectory +play.modules { + # By default, Play will load any class called Module that is defined + # in the root package (the "app" directory), or you can define them + # explicitly below. + # If there are any built-in modules that you want to enable, you can list them here. + enabled += modules.TaxonomyModule + + # If there are any built-in modules that you want to disable, you can list them here. + #disabled += "" +} + +## IDE +# https://www.playframework.com/documentation/latest/IDE +# ~~~~~ +# Depending on your IDE, you can add a hyperlink for errors that will jump you +# directly to the code location in the IDE in dev mode. The following line makes +# use of the IntelliJ IDEA REST interface: +#play.editor="http://localhost:63342/api/file/?file=%s&line=%s" + +## Internationalisation +# https://www.playframework.com/documentation/latest/JavaI18N +# https://www.playframework.com/documentation/latest/ScalaI18N +# ~~~~~ +# Play comes with its own i18n settings, which allow the user's preferred language +# to map through to internal messages, or allow the language to be stored in a cookie. +play.i18n { + # The application languages + langs = [ "en" ] + + # Whether the language cookie should be secure or not + #langCookieSecure = true + + # Whether the HTTP only attribute of the cookie should be set to true + #langCookieHttpOnly = true +} + +## Play HTTP settings +# ~~~~~ +play.http { + ## Router + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # Define the Router object to use for this application. + # This router will be looked up first when the application is starting up, + # so make sure this is the entry point. + # Furthermore, it's assumed your route file is named properly. + # So for an application router like `my.application.Router`, + # you may need to define a router file `conf/my.application.routes`. + # Default to Routes in the root package (aka "apps" folder) (and conf/routes) + #router = my.application.Router + + ## Action Creator + # https://www.playframework.com/documentation/latest/JavaActionCreator + # ~~~~~ + #actionCreator = null + + ## ErrorHandler + # https://www.playframework.com/documentation/latest/JavaRouting + # https://www.playframework.com/documentation/latest/ScalaRouting + # ~~~~~ + # If null, will attempt to load a class called ErrorHandler in the root package, + #errorHandler = null + + ## Session & Flash + # https://www.playframework.com/documentation/latest/JavaSessionFlash + # https://www.playframework.com/documentation/latest/ScalaSessionFlash + # ~~~~~ + session { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + + # Sets the max-age field of the cookie to 5 minutes. + # NOTE: this only sets when the browser will discard the cookie. Play will consider any + # cookie value with a valid signature to be a valid session forever. To implement a server side session timeout, + # you need to put a timestamp in the session and check it at regular intervals to possibly expire it. + #maxAge = 300 + + # Sets the domain on the session cookie. + #domain = "example.com" + } + + flash { + # Sets the cookie to be sent only over HTTPS. + #secure = true + + # Sets the cookie to be accessed only by the server. + #httpOnly = true + } +} + +play.server.http.idleTimeout = 60s +play.http.parser.maxDiskBuffer = 10MB +parsers.anyContent.maxLength = 10MB + +## Netty Provider +# https://www.playframework.com/documentation/latest/SettingsNetty +# ~~~~~ +play.server.netty { + # Whether the Netty wire should be logged + log.wire = true + + # If you run Play on Linux, you can use Netty's native socket transport + # for higher performance with less garbage. + transport = "native" +} + +## WS (HTTP Client) +# https://www.playframework.com/documentation/latest/ScalaWS#Configuring-WS +# ~~~~~ +# The HTTP client primarily used for REST APIs. The default client can be +# configured directly, but you can also create different client instances +# with customized settings. You must enable this by adding to build.sbt: +# +# libraryDependencies += ws // or javaWs if using java +# +play.ws { + # Sets HTTP requests not to follow 302 requests + #followRedirects = false + + # Sets the maximum number of open HTTP connections for the client. + #ahc.maxConnectionsTotal = 50 + + ## WS SSL + # https://www.playframework.com/documentation/latest/WsSSL + # ~~~~~ + ssl { + # Configuring HTTPS with Play WS does not require programming. You can + # set up both trustManager and keyManager for mutual authentication, and + # turn on JSSE debugging in development with a reload. + #debug.handshake = true + #trustManager = { + # stores = [ + # { type = "JKS", path = "exampletrust.jks" } + # ] + #} + } +} + +## Cache +# https://www.playframework.com/documentation/latest/JavaCache +# https://www.playframework.com/documentation/latest/ScalaCache +# ~~~~~ +# Play comes with an integrated cache API that can reduce the operational +# overhead of repeated requests. You must enable this by adding to build.sbt: +# +# libraryDependencies += cache +# +play.cache { + # If you want to bind several caches, you can bind the individually + #bindCaches = ["db-cache", "user-cache", "session-cache"] +} + +## Filter Configuration +# https://www.playframework.com/documentation/latest/Filters +# ~~~~~ +# There are a number of built-in filters that can be enabled and configured +# to give Play greater security. +# +play.filters { + + # Enabled filters are run automatically against Play. + # CSRFFilter, AllowedHostFilters, and SecurityHeadersFilters are enabled by default. + enabled = [filters.AccessLogFilter] + + # Disabled filters remove elements from the enabled list. + # disabled += filters.CSRFFilter + + + ## CORS filter configuration + # https://www.playframework.com/documentation/latest/CorsFilter + # ~~~~~ + # CORS is a protocol that allows web applications to make requests from the browser + # across different domains. + # NOTE: You MUST apply the CORS configuration before the CSRF filter, as CSRF has + # dependencies on CORS settings. + cors { + # Filter paths by a whitelist of path prefixes + #pathPrefixes = ["/some/path", ...] + + # The allowed origins. If null, all origins are allowed. + #allowedOrigins = ["http://www.example.com"] + + # The allowed HTTP methods. If null, all methods are allowed + #allowedHttpMethods = ["GET", "POST"] + } + + ## Security headers filter configuration + # https://www.playframework.com/documentation/latest/SecurityHeaders + # ~~~~~ + # Defines security headers that prevent XSS attacks. + # If enabled, then all options are set to the below configuration by default: + headers { + # The X-Frame-Options header. If null, the header is not set. + #frameOptions = "DENY" + + # The X-XSS-Protection header. If null, the header is not set. + #xssProtection = "1; mode=block" + + # The X-Content-Type-Options header. If null, the header is not set. + #contentTypeOptions = "nosniff" + + # The X-Permitted-Cross-Domain-Policies header. If null, the header is not set. + #permittedCrossDomainPolicies = "master-only" + + # The Content-Security-Policy header. If null, the header is not set. + #contentSecurityPolicy = "default-src 'self'" + } + + ## Allowed hosts filter configuration + # https://www.playframework.com/documentation/latest/AllowedHostsFilter + # ~~~~~ + # Play provides a filter that lets you configure which hosts can access your application. + # This is useful to prevent cache poisoning attacks. + hosts { + # Allow requests to example.com, its subdomains, and localhost:9000. + #allowed = [".example.com", "localhost:9000"] + } +} + +play.http.parser.maxMemoryBuffer = 50MB +akka.http.parsing.max-content-length = 50MB +schema.base_path="{{ kp_schema_base_path | default('/home/sunbird/taxonomy-service-1.0-SNAPSHOT/schemas')}}" + +# Cassandra Configuration +cassandra { + lp { + connection: "{{ lp_cassandra_connection }}" + } + lpa { + connection: "{{ dp_cassandra_connection }}" + } +} + +# Redis Configuration +redis { + host: "{{ sunbird_lp_redis_host }}" + port: 6379 + maxConnections: 128 +} + +# Configuration +akka.request_timeout: 30 +environment.id: {{ environment_id }} +graph { + dir: "/data/graphDB" + ids: ["domain"] + passport.key.base: "{{ graph_passport_key }}" +} +route { + domain: "{{ learning_neo4j_bolt_url }}" + all: "{{ language_neo4j_bolt_url }}" + bolt { + write { + domain: "{{ learning_write_elb_url }}" + all: "{{ language_write_elb_url }}" + }, + read { + domain: "{{ learning_read_elb_url }}" + all: "{{ language_read_elb_url }}" + } + } +} + +shard.id: {{ mw_shard_id }} +platform { + auth.check.enabled: false + cache.ttl: 3600000 +} + +# Cloud Storage Config +cloud_storage_type: "azure" +azure_storage_key: "{{ sunbird_public_storage_account_name }}" +azure_storage_secret: "{{ sunbird_public_storage_account_key }}" +azure_storage_container: "{{ sunbird_content_azure_storage_container }}" + +installation.id: ekstep + +channel { + default: "in.ekstep" +} + +languageCode { + assamese : "as" + bengali : "bn" + english : "en" + gujarati : "gu" + hindi : "hi" + kannada : "ka" + marathi : "mr" + odia : "or" + tamil : "ta" + telugu : "te" +} +objectcategorydefinition.keyspace="{{ lp_cassandra_keyspace_prefix }}_category_store" diff --git a/ansible/roles/stack-sunbird/templates/taxonomy-service_logback.xml b/ansible/roles/stack-sunbird/templates/taxonomy-service_logback.xml new file mode 100644 index 000000000..1d3fdd0c6 --- /dev/null +++ b/ansible/roles/stack-sunbird/templates/taxonomy-service_logback.xml @@ -0,0 +1,60 @@ + + + + + + + + + + %d %msg%n + + + + + + + + + + + + + + + + + + + + %msg + + + {{env_name}}.telemetry.raw + + + + + + + + + bootstrap.servers={{lp_kafka_url}} + + acks=0 + + linger.ms=15000 + + max.block.ms=0 + + client.id=${HOSTNAME}-${CONTEXT_NAME}-logback-relaxed + + + + + + + + + + \ No newline at end of file diff --git a/ansible/roles/stack-sunbird/templates/telemetry-logstash-datapipeline.conf b/ansible/roles/stack-sunbird/templates/telemetry-logstash-datapipeline.conf index e2e035ad0..3d9df7d5f 100644 --- a/ansible/roles/stack-sunbird/templates/telemetry-logstash-datapipeline.conf +++ b/ansible/roles/stack-sunbird/templates/telemetry-logstash-datapipeline.conf @@ -5,6 +5,8 @@ input { codec => "json" topics => ["{{sunbird_telemetry_kafka_topic}}"] retry_backoff_ms => 180000 + consumer_threads => 4 + max_poll_records => "100" max_partition_fetch_bytes => "5242880" } } diff --git a/ansible/ssh.yml b/ansible/ssh.yml index fb5da2d9a..a4e02d323 100644 --- a/ansible/ssh.yml +++ b/ansible/ssh.yml @@ -1,12 +1,12 @@ --- -- hosts: "{{ remote }}" +- hosts: "bastion:{{ remote }}" become: yes roles: - add_user tags: - add-user -- hosts: "{{ remote }}" +- hosts: "bastion:{{ remote }}" become: yes tasks: - user: name="{{ ssh_user }}" state=absent remove=yes diff --git a/deploy/cassandra_backup.py b/deploy/cassandra_backup.py index 076379a9d..6936b862f 100755 --- a/deploy/cassandra_backup.py +++ b/deploy/cassandra_backup.py @@ -19,28 +19,45 @@ for help ./cassandra_backup.py -h ''' -from os import walk, sep, system, getcwd, makedirs +from os import walk, sep, system, getcwd, makedirs, cpu_count, link, path, makedirs from argparse import ArgumentParser from shutil import rmtree, ignore_patterns, copytree from re import match, compile from sys import exit -from tempfile import mkdtemp from time import strftime +import concurrent.futures +import errno + +# Create temporary directory to copy data +default_snapshot_name = "cassandra_backup" + strftime("%Y-%m-%d-%H%M%S") +tmpdir = getcwd()+sep+default_snapshot_name parser = ArgumentParser(description="Create a snapshot and create tar ball inside tardirectory") parser.add_argument("-d", "--datadirectory", metavar="datadir", default='/var/lib/cassandra/data', help="Path to cassadandra keyspaces. Default /var/lib/cassadra/data") -parser.add_argument("-s", "--snapshotname", metavar="snapshotname", +parser.add_argument("-s", "--snapshotdirectory", metavar="snapdir", default=tmpdir, + help="Path to take cassandra snapshot. Default {}".format(tmpdir)) +parser.add_argument("-n", "--snapshotname", metavar="snapshotname", default="cassandra_backup-"+strftime("%Y-%m-%d"), - help="Name with which snapshot to be taken. Default {}".format("cassandra_backup-"+strftime("%Y-%m-%d"))) + help="Name with which snapshot to be taken. Default {}".format(default_snapshot_name)) parser.add_argument("-t", "--tardirectory", metavar="tardir", - default=getcwd(), help="Path to create the tarball. Default {}".format(getcwd())) + default='', help="Path to create the tarball. Disabled by Default") +parser.add_argument("-w", "--workers", metavar="workers", + default=cpu_count(), help="Number of workers to use. Default same as cpu cores {}".format(cpu_count())) +parser.add_argument("--disablesnapshot", action="store_true", + help="disable taking snapshot, snapshot name can be given via -s flag") args = parser.parse_args() -# Create temporary directory to copy data -tmpdir = mkdtemp() -makedirs(tmpdir+sep+"cassandra_backup") +tmpdir = args.snapshotdirectory +# Trying to create the directory if not exists +try: + makedirs(tmpdir+sep+"cassandra_backup") +except OSError as e: + raise +def customCopy(root, root_target_dir): + print("copying {} to {}".format(root, root_target_dir)) + copytree(src=root, dst=root_target_dir, copy_function=link, ignore=ignore_patterns('.*')) def copy(): ''' @@ -48,17 +65,26 @@ def copy(): ''' root_levels = args.datadirectory.rstrip('/').count(sep) ignore_list = compile(tmpdir+sep+"cassandra_backup"+sep+'(system|system|systemtauth|system_traces|system_schema|system_distributed)') - + # List of the threds running in background + futures = [] try: - for root, dirs, files in walk(args.datadirectory): - root_target_dir = tmpdir+sep+"cassandra_backup"+sep+sep.join(root.split(sep)[root_levels+1:-2]) - if match(ignore_list, root_target_dir): - continue - if root.split(sep)[-1] == args.snapshotname: - copytree(src=root, dst=root_target_dir, ignore=ignore_patterns('.*')) + with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as executor: + for root, dirs, files in walk(args.datadirectory): + root_target_dir = tmpdir+sep+"cassandra_backup"+sep+sep.join(root.split(sep)[root_levels+1:-2]) + if match(ignore_list, root_target_dir): + continue + if root.split(sep)[-1] == args.snapshotname: + # Keeping copy operation in background with threads + tmp_arr = [root, root_target_dir] + futures.append( executor.submit( lambda p: customCopy(*p), tmp_arr)) except Exception as e: print(e) - + # Checking status of the copy operation + for future in concurrent.futures.as_completed(futures): + try: + print("Task completed. Result: {}".format(future.result())) + except Exception as e: + print(e) # Creating schema command = "cqlsh -e 'DESC SCHEMA' > {}/cassandra_backup/db_schema.cql".format(tmpdir) @@ -67,18 +93,42 @@ def copy(): print("Couldn't backup schema, exiting...") exit(1) print("Schema backup completed. saved in {}/cassandra_backup/db_schema.sql".format(tmpdir)) -# Cleaning all old snapshots -command = "nodetool clearsnapshot" -system(command) + # Creating snapshots -command = "nodetool snapshot -t {}".format(args.snapshotname) -rc = system(command) -if rc == 0: +if not args.disablesnapshot: + # Cleaning all old snapshots + command = "nodetool clearsnapshot" + system(command) + # Taking new snapshot + command = "nodetool snapshot -t {}".format(args.snapshotname) + rc = system(command) + if rc != 0: + print("Backup failed") + exit(1) print("Snapshot taken.") - copy() + +# Copying the snapshot to proper folder structure, this is not a copy but a hard link +copy() + +# Clearing the snapshot. +# We've the data now available in the copied directory. +command = "nodetool clearsnapshot -t {}".format(args.snapshotname) +print("Clearing snapshot {} ...".format(args.snapshotname)) +rc = system(command) +if rc != 0: + print("Clearing snapshot {} failed".format(args.snapshotname)) + raise + exit(1) + +# Creating tarball +if args.tardirectory: print("Making a tarball: {}.tar.gz".format(args.snapshotname)) - command = "cd {} && tar -czvf {}/{}.tar.gz *".format(tmpdir, args.tardirectory, args.snapshotname) - system(command) + command = "cd {} && tar --remove-files -czvf {}/{}.tar.gz *".format(tmpdir, args.tardirectory, args.snapshotname) + rc = system(command) + if rc != 0: + print("Creation of tar failed") + exit(1) # Cleaning up backup directory rmtree(tmpdir) print("Cassandra backup completed and stored in {}/{}.tar.gz".format(args.tardirectory, args.snapshotname)) + diff --git a/deploy/grafana/backup.sh b/deploy/grafana/backup.sh new file mode 100755 index 000000000..d37c4287d --- /dev/null +++ b/deploy/grafana/backup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# How to run +# docker run --rm -v /etc/passwd:/etc/passwd -u $(id -u):$(id -g) -v$(pwd):/work -v $(pwd)/backup.sh:/work/backup.sh rjshrjndrn/wizzy /work/backup.sh https://grafna/url admin password +url=$1 +username=$2 +password=$3 +mode=${4:-import} +item=${5:-dashboards} +script=$(echo $0) +cd $(dirname "$script") +echo ${mode}ing grafafna in $(dirname "$script") +wizzy init +wizzy set grafana url $url +wizzy set grafana username $username +wizzy set grafana password $password +wizzy $mode $item diff --git a/deploy/grafana/dashboards/General/alerts.json b/deploy/grafana/dashboards/General/alerts.json new file mode 100644 index 000000000..c33bfc181 --- /dev/null +++ b/deploy/grafana/dashboards/General/alerts.json @@ -0,0 +1,144 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": 426, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ALERTS{alertstate=\"firing\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{alertname}}-{{instance}}-{{job}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Alerts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "1", + "min": "0", + "show": false + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Alerts", + "uid": "POm4CxmWk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/api-manager.json b/deploy/grafana/dashboards/General/api-manager.json new file mode 100644 index 000000000..8ae903c83 --- /dev/null +++ b/deploy/grafana/dashboards/General/api-manager.json @@ -0,0 +1,870 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1558355502529, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(kong_request_count{api=~\"${api}\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 9, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_latency_time{api=~\"${api}\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_upstream_latency_time{api=~\"${api}\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Upstream Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 10, + "panels": [], + "repeat": null, + "title": "Status Code", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{api=~\"${api}\",status_code=~\"2..|3..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count{api=~\"${api}\"}[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Success %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{api=~\"${api}\",status_code!~\"2..|3..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count{api=~\"${api}\"}[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 11, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_request_size{api=~\"${api}\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_response_size{api=~\"${api}\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "prom-fed", + "definition": "label_values(kong_request_count,api)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "api", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "assemblePage", + "value": "assemblePage" + }, + { + "selected": false, + "text": "assignRole", + "value": "assignRole" + }, + { + "selected": false, + "text": "compositeSearch", + "value": "compositeSearch" + }, + { + "selected": false, + "text": "createUserVersion2", + "value": "createUserVersion2" + }, + { + "selected": false, + "text": "deviceRegister", + "value": "deviceRegister" + }, + { + "selected": false, + "text": "getCourseHierarchy", + "value": "getCourseHierarchy" + }, + { + "selected": false, + "text": "getUserByKey", + "value": "getUserByKey" + }, + { + "selected": false, + "text": "getUserProfile", + "value": "getUserProfile" + }, + { + "selected": false, + "text": "readChannel", + "value": "readChannel" + }, + { + "selected": false, + "text": "readContent", + "value": "readContent" + }, + { + "selected": false, + "text": "readForm", + "value": "readForm" + }, + { + "selected": false, + "text": "readFramework", + "value": "readFramework" + }, + { + "selected": false, + "text": "searchContent", + "value": "searchContent" + }, + { + "selected": false, + "text": "searchOrg", + "value": "searchOrg" + }, + { + "selected": false, + "text": "searchUser", + "value": "searchUser" + }, + { + "selected": false, + "text": "telemetry", + "value": "telemetry" + } + ], + "query": "label_values(kong_request_count,api)", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "API Manager", + "uid": "000000008" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/availability.json b/deploy/grafana/dashboards/General/availability.json new file mode 100644 index 000000000..942f63d0e --- /dev/null +++ b/deploy/grafana/dashboards/General/availability.json @@ -0,0 +1,333 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_success{job=~\"availability.*\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Availability", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "1", + "min": "0", + "show": false + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_duration_seconds", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Probe Time", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_http_status_code", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Probe HTTP status code", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Availability", + "uid": "rzAQjbmZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/container-details-ds1.json b/deploy/grafana/dashboards/General/container-details-ds1.json new file mode 100644 index 000000000..33c2ef168 --- /dev/null +++ b/deploy/grafana/dashboards/General/container-details-ds1.json @@ -0,0 +1,999 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Docker monitoring with Prometheus and cAdvisor", + "editable": true, + "gnetId": 193, + "graphTooltip": 1, + "iteration": 1558350683385, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS1", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "100", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(container_last_seen{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_last_seen", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Running containers", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS1", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "height": "100", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_cpu_system_seconds_total) / ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Total CPU Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS1", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "height": "100", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus_stateful", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "height": "100", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}) / sum(node_memory_MemTotal_bytes{job=\"vm-node-exporter\"}) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage %", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~'$service'}[5m])) by (container_label_com_docker_swarm_service_name)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{container_label_com_docker_swarm_service_name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"} / container_spec_memory_limit_bytes) < Inf", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage %", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Rx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Tx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus_DS1", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Container Details DS1", + "uid": "2pyUniiZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/container-details-ds2.json b/deploy/grafana/dashboards/General/container-details-ds2.json new file mode 100644 index 000000000..ecba47699 --- /dev/null +++ b/deploy/grafana/dashboards/General/container-details-ds2.json @@ -0,0 +1,1005 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Docker monitoring with Prometheus and cAdvisor", + "editable": true, + "gnetId": 193, + "graphTooltip": 1, + "iteration": 1557463511522, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS2", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "100", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(container_last_seen{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_last_seen", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Running containers", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus_stateful", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "height": "100", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m]) * 100) / sum(count by (instance)(node_cpu_seconds_total{mode=\"system\", job=\"vm-node-exporter\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "", + "title": "Total CPU Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS2", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "height": "100", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus_stateful", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "height": "100", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}) / sum(node_memory_MemTotal_bytes{job=\"vm-node-exporter\"}) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage %", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~'$service'}[5m])) by (container_label_com_docker_swarm_service_name)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"} / container_spec_memory_limit_bytes) < Inf", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage %", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Rx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS2", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Tx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus_DS2", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Container Details DS2", + "uid": "zWYVQbiZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/container-details-ds3.json b/deploy/grafana/dashboards/General/container-details-ds3.json new file mode 100644 index 000000000..91be5dbe6 --- /dev/null +++ b/deploy/grafana/dashboards/General/container-details-ds3.json @@ -0,0 +1,1005 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Docker monitoring with Prometheus and cAdvisor", + "editable": true, + "gnetId": 193, + "graphTooltip": 1, + "iteration": 1558028798889, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS3", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "100", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(container_last_seen{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_last_seen", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Running containers", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus_stateful", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "height": "100", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m]) * 100) / sum(count by (instance)(node_cpu_seconds_total{mode=\"system\", job=\"vm-node-exporter\"}))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "", + "title": "Total CPU Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_DS3", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "height": "100", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prometheus_stateful", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "height": "100", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}) / sum(node_memory_MemTotal_bytes{job=\"vm-node-exporter\"}) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": "", + "title": "Total Memory Usage %", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~'$service'}[5m])) by (container_label_com_docker_swarm_service_name)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Container CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(container_memory_usage_bytes{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"} / container_spec_memory_limit_bytes) < Inf", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_memory_usage_bytes", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage %", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "container_network_receive_bytes_total", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Rx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS3", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Tx", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "docker" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus_DS3", + "definition": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Container Details DS3", + "uid": "38mOQxmZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/coredns.json b/deploy/grafana/dashboards/General/coredns.json new file mode 100644 index 000000000..d669fc3dd --- /dev/null +++ b/deploy/grafana/dashboards/General/coredns.json @@ -0,0 +1,1306 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server.", + "editable": true, + "gnetId": 5926, + "graphTooltip": 0, + "iteration": 1539947521873, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (zone)", + "intervalFactor": 2, + "legendFormat": "{{zone}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{instance=~\"$instance\"}[5m])) by (rcode)", + "intervalFactor": 2, + "legendFormat": "{{rcode}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le, job))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "hits:{{type}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "selected": true, + "tags": [], + "text": "172.16.1.8:9153", + "value": "172.16.1.8:9153" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "up{job=\"coredns\"}", + "refresh": 1, + "regex": ".*instance=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "CoreDNS", + "uid": "vkQ0UHxik" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/druid.json b/deploy/grafana/dashboards/General/druid.json new file mode 100644 index 000000000..3141d3588 --- /dev/null +++ b/deploy/grafana/dashboards/General/druid.json @@ -0,0 +1,252 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "Prometheus_Stateful", + "fontSize": "120%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "__name__", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": "value", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "#37872D", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Value", + "thresholds": [ + "99" + ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "job", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "http_druid_health_check_status", + "format": "table", + "instant": true, + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "DRUID CURRENT AVAILABILITY", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "http_druid_health_check_status", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ datasource }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Druid Datasource Availability", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Druid", + "uid": "XWxgcjvZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/elasticsearch.json b/deploy/grafana/dashboards/General/elasticsearch.json new file mode 100644 index 000000000..f9f2244e3 --- /dev/null +++ b/deploy/grafana/dashboards/General/elasticsearch.json @@ -0,0 +1,1523 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "iteration": 1558351428419, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "elasticsearch_cluster_health_number_of_nodes", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Running Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Delayed Unassigned Shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Initialising Shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Unassigned Shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(elasticsearch_thread_pool_active_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Type: {{ type }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Thread Pools", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "bucketAggs": [ + { + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "dsType": "elasticsearch", + "expr": "avg(irate(elasticsearch_process_cpu_percent{cluster=~\"$cluster\"}[5m])) by(name) *100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }}", + "metric": "elasticsearch_breakers_tripped", + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "refId": "A", + "step": 240, + "timeField": "failure_tstamp" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(elasticsearch_indices_search_query_time_seconds{cluster=~\"$cluster\"}[5m])) by (name)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. Query Time (last 5m)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(elasticsearch_indices_search_fetch_time_seconds{cluster=~\"$cluster\"}[5m])) by (name)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg. Fetch Time (last 5m)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(elasticsearch_indices_search_query_total{cluster=~\"$cluster\"}) by (name)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# of Queries", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(elasticsearch_indices_search_fetch_total{cluster=~\"$cluster\"}) by (name)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "# of Fetches", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(elasticsearch_indices_docs{cluster=~\"$cluster\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Documents", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents indexed", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Index Size", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Index Size", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(elasticsearch_transport_rx_packets_total{cluster=~\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "RX", + "refId": "A", + "step": 240 + }, + { + "expr": "sum(rate(elasticsearch_transport_tx_packets_total{cluster=~\"$cluster\"}[5m])) * -1", + "intervalFactor": 2, + "legendFormat": "TX", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RX/TX Rate 5m", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg_over_time(elasticsearch_jvm_memory_used_bytes{area=\"heap\",cluster=~\"$cluster\"}[15m]) / elasticsearch_jvm_memory_max_bytes{area=\"heap\",cluster=~\"$cluster\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Avg Heap in 15min", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "", + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} {{ gc }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "GC seconds", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\", type!=\"management\"}[5m])", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} {{ type }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Thread pool rejections", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "1m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "elastic" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "sunbird-dev", + "value": "sunbird-dev" + }, + "datasource": "Prometheus_Stateful", + "definition": "label_values(elasticsearch_cluster_health_number_of_data_nodes,cluster)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(elasticsearch_cluster_health_number_of_data_nodes,cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "ElasticSearch", + "uid": "KddljxiWk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/ep-pipeline-consumerlag.json b/deploy/grafana/dashboards/General/ep-pipeline-consumerlag.json new file mode 100644 index 000000000..b08ae5057 --- /dev/null +++ b/deploy/grafana/dashboards/General/ep-pipeline-consumerlag.json @@ -0,0 +1,670 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TelemetryExtractor- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 8 + }, + "id": 4, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TelemetryLocationUpdater- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 16 + }, + "id": 6, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "consumer_lag" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "TelemetryValidator" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "sunbirddev" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TelemetryValidator- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 24 + }, + "id": 5, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "consumer_lag" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "DeDuplication" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "sunbirddev" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DeDuplication- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 32 + }, + "id": 7, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "consumer_lag" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "TelemetryRouter" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "sunbirddev" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TelemetryRouter- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 40 + }, + "id": 8, + "legend": { + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "consumer_lag" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "DeNormalization" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "sunbirddev" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DeNormalization- Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "EP-Pipeline - ConsumerLag", + "uid": "I5-sVKmZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/ep-pipeline-consumerlag1.json b/deploy/grafana/dashboards/General/ep-pipeline-consumerlag1.json new file mode 100644 index 000000000..9bb35657c --- /dev/null +++ b/deploy/grafana/dashboards/General/ep-pipeline-consumerlag1.json @@ -0,0 +1,789 @@ +{ + "annotations": { + "list": [] + }, + "editMode": false, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [], + "intervalFactor": 2, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryExtractor' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "consumer_lag" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=", + "value": "ntpprod" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryExtractor" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TelemtryExtractor-lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryLocationUpdater' AND \"env\" = 'sunbirddev') AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryLocationUpdater' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 60, + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TelemtryLocationUpdater-lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryValidator' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TelemtryValidator-lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'DeDuplication' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DeDuplication-lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryRouter' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "TelemetryRouter" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "ntpprod" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TelemtryRouter-lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 250, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'DeNormalization' AND \"env\" = 'ntpprod') ", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"consumer_lag\" FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'DeNormalization' AND \"env\" = 'ntpprod') AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 4, + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DeNormalization-Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "EP-Pipeline-ConsumerLag1", + "uid": "mHlLByWZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/ep-pipeline-metrics.json b/deploy/grafana/dashboards/General/ep-pipeline-metrics.json new file mode 100644 index 000000000..b6bde7248 --- /dev/null +++ b/deploy/grafana/dashboards/General/ep-pipeline-metrics.json @@ -0,0 +1,4136 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1558372314963, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(40, 237, 42, 0.89)", + "rgba(23, 226, 15, 0.97)" + ], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'TelemetryValidator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "Telemetry validation success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'TelemetryValidator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "Telemetry validation failures", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(145, 23, 23, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'DeDuplication' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "Telemetry Dedup failures", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(31, 195, 25, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(145, 23, 23, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(11, 127, 221, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'DeDuplication' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": "", + "title": "Unique Events", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "total" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 5, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"success\", \"skipped\" FROM \"pipeline_metrics\" WHERE \"env\" = 'ntpprod' AND \"job_name\" = 'TelemetryValidator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=", + "value": "production" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryValidator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Validator Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#E0752D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 6, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryValidator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Validator Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "policy": "autogen", + "query": "SELECT \"success\", \"skipped\" FROM \"autogen\".\"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'TelemetryConverter' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "DeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Converter Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryConverter" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Converter Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 9, + "interval": ">300s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "DeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DeDuplication Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#EF843C" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 10, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "DeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "DeDuplication Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 11, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "ReverseSearch" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ReverseSearch Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#E0752D", + "pipeline_metrics.failed": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 12, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "ReverseSearch" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ReverseSearch Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 13, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "ObjectDeNormalization" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ObjectDeNormalization Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 14, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "ObjectDeNormalization" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ObjectDeNormalization Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 15, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PublicExhaustDeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Public Exhaust De Duplication Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 16, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PublicExhaustDeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Public Exhaust De Duplication Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 17, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PrivateExhaustDeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Private Exhaust De Duplication Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 18, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PrivateExhaustDeDuplication" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Private Exhaust De Duplication Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 19, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PartnerDataRouter" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Partner Data Router Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 20, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PartnerDataRouter" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Partner Data Router Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PublicTelemetryTask" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Public Telemetry Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 22, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "PublicTelemetryTask" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Public Telemetry Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 23, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsRouterPrimary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Router Primary Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 24, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsRouterPrimary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Router Primary Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 25, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsRouterSecondary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Router Secondary Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 26, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsRouterSecondary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Router Secondary Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 27, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsIndexerPrimary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Indexer Primary Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 28, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsIndexerSecondary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Indexer Primary Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 29, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsIndexerSecondary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Indexer Secondary Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#BF1B00", + "pipeline_metrics.failed": "#F9934E" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 30, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ], + [ + { + "params": [ + "error" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "EsIndexerSecondary" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ES Indexer Secondary Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 107 + }, + "id": 31, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"processed\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'TelemetryLocationUpdater' AND \"env\" = 'ntpprod') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "processed" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "TelemetryLocationUpdater" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "ntpprod" + } + ] + } + ], + "thresholds": "", + "title": "Processed message For Device Location Updater", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 114 + }, + "id": 32, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"db_error\") FROM \"pipeline_metrics\" WHERE (\"env\" = 'ntpprod' AND \"job_name\" = 'TelemetryLocationUpdater') AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "db_error" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "job_name", + "operator": "=", + "value": "TelemetryLocationUpdater" + }, + { + "condition": "AND", + "key": "env", + "operator": "=", + "value": "ntpprod" + } + ] + } + ], + "thresholds": "", + "title": "Cassandra DB error for Telemetry Location Updater", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 121 + }, + "id": 33, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "cache_empty_values" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=", + "value": "ntpprod" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryLocationUpdater" + } + ] + } + ], + "thresholds": "", + "title": "cache empty values for device location", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pipeline", + "home" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "ntpprod", + "value": "ntpprod" + }, + "datasource": "Monitoring Events", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "tagValuesQuery": "env", + "tagsQuery": "pipeline_metrics", + "type": "custom", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "EP Pipeline Metrics", + "uid": "WMF0upWZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/etcd.json b/deploy/grafana/dashboards/General/etcd.json new file mode 100644 index 000000000..81a623aa8 --- /dev/null +++ b/deploy/grafana/dashboards/General/etcd.json @@ -0,0 +1,1092 @@ +{ + "annotations": { + "list": [] + }, + "description": "etcd sample Grafana dashboard with Prometheus", + "editable": true, + "gnetId": null, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "$datasource", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "sum(etcd_server_has_leader{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "", + "metric": "etcd_server_has_leader", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Up", + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 23, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_started_total{job=\"$cluster\",grpc_type=\"unary\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Rate", + "metric": "grpc_server_started_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(grpc_server_handled_total{job=\"$cluster\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RPC Failed Rate", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 41, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Watch Streams", + "metric": "grpc_server_handled_total", + "refId": "A", + "step": 4 + }, + { + "expr": "sum(grpc_server_started_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"$cluster\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", + "intervalFactor": 2, + "legendFormat": "Lease Streams", + "metric": "grpc_server_handled_total", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Active Streams", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "showTitle": false, + "title": "Row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"$cluster\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB Size", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "DB Size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{instance}} WAL fsync", + "metric": "etcd_disk_wal_fsync_duration_seconds_bucket", + "refId": "A", + "step": 4 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"$cluster\"}[5m])) by (instance, le))", + "intervalFactor": 2, + "legendFormat": "{{instance}} DB fsync", + "metric": "etcd_disk_backend_commit_duration_seconds_bucket", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Sync Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 29, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "{{instance}} Resident Memory", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 22, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic In", + "metric": "etcd_network_client_grpc_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 5, + "id": 21, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"$cluster\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Client Traffic Out", + "metric": "etcd_network_client_grpc_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 20, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic In", + "metric": "etcd_network_peer_received_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic In", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": null, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"$cluster\"}[5m])) by (instance)", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{instance}} Peer Traffic Out", + "metric": "etcd_network_peer_sent_bytes_total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Peer Traffic Out", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "editable": true, + "error": false, + "fill": 0, + "id": 40, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Failure Rate", + "metric": "etcd_server_proposals_failed_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(etcd_server_proposals_pending{job=\"$cluster\"})", + "intervalFactor": 2, + "legendFormat": "Proposal Pending Total", + "metric": "etcd_server_proposals_pending", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Commit Rate", + "metric": "etcd_server_proposals_committed_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "Proposal Apply Rate", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Raft Proposals", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "$datasource", + "decimals": 0, + "editable": true, + "error": false, + "fill": 0, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "changes(etcd_server_leader_changes_seen_total{job=\"$cluster\"}[1d])", + "intervalFactor": 2, + "legendFormat": "{{instance}} Total Leader Elections Per Day", + "metric": "etcd_server_leader_changes_seen_total", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Leader Elections Per Day", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "title": "New row" + } + ], + "schemaVersion": 13, + "sharedCrosshair": false, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(etcd_server_has_leader, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "etcd", + "uid": "P0AdkUEZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/host-details.json b/deploy/grafana/dashboards/General/host-details.json new file mode 100644 index 000000000..f26364ebb --- /dev/null +++ b/deploy/grafana/dashboards/General/host-details.json @@ -0,0 +1,1524 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A simple overview of the most important Docker metrics export by node-expoter on docker swarm mode. ", + "editable": true, + "gnetId": 2666, + "graphTooltip": 0, + "iteration": 1575095285923, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": null, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "nodename", + "targets": [ + { + "expr": "node_uname_info{instance=~\"$server\"}", + "format": "table", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Host", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_load1{instance='$server'} / count by (job, instance)(count by(job, instance, cpu)(node_cpu_seconds_total{instance='$server'}))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0.8,0.9", + "title": "Load", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 0 + }, + "hideTimeOverride": true, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_memory_MemAvailable_bytes{instance=~\"$server\"}", + "format": "time_series", + "interval": "30s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "timeFrom": "10s", + "title": "Available Memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 0 + }, + "hideTimeOverride": true, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(node_filesystem_free_bytes{fstype=~\"xfs|ext4\",instance=~\"$server\"})", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 30 + } + ], + "thresholds": "", + "timeFrom": "10s", + "title": "Free Storage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 1, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time() - node_boot_time_seconds{instance=~\"$server\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "hideTimeOverride": true, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$server\"}[5m])) * 100 / count(node_cpu_seconds_total{mode=\"user\", instance=~\"$server\"})", + "format": "time_series", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 20 + } + ], + "thresholds": "", + "timeFrom": "10s", + "title": "CPU Idle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 3, + "grid": {}, + "gridPos": { + "h": 5, + "w": 10, + "x": 0, + "y": 3 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg without (cpu) (irate(node_cpu_seconds_total{instance=~\"$server\", mode!=\"idle\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mode}}", + "refId": "A", + "step": 4, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": {}, + "gridPos": { + "h": 5, + "w": 14, + "x": 10, + "y": 3 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{device!=\"lo\",instance=~\"$server\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "In: {{ device }}", + "metric": "node_network_receive_bytes", + "refId": "A", + "step": 2 + }, + { + "expr": "irate(node_network_transmit_bytes_total{device!=\"lo\",instance=~\"$server\"}[5m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Out: {{ device }}", + "metric": "node_network_transmit_bytes", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 20, + "x": 0, + "y": 8 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Free", + "zindex": 3 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$server\"} - node_memory_MemFree_bytes{instance=~\"$server\"} - node_memory_Cached_bytes{instance=~\"$server\"} - node_memory_Buffers_bytes{instance=~\"$server\"} - node_memory_Slab_bytes{instance=~\"$server\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Used", + "metric": "memo", + "refId": "A", + "step": 4, + "target": "" + }, + { + "expr": "node_memory_Buffers_bytes{instance=~\"$server\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Buffers", + "refId": "C", + "step": 4 + }, + { + "expr": "node_memory_Cached_bytes{instance=~\"$server\"} + node_memory_Slab_bytes{instance=~\"$server\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cached", + "refId": "D", + "step": 4 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~\"$server\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Free", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 8 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(node_memory_MemAvailable_bytes{instance=~\"$server\"} / node_memory_MemTotal_bytes{instance=~\"$server\"}) * 100", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 40, + "target": "" + } + ], + "thresholds": "10, 20", + "title": "Available memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 10, + "x": 0, + "y": 13 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "{instance=\"172.17.0.1:9100\"}", + "yaxis": 2 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (instance) (irate(node_disk_reads_completed_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 4, + "legendFormat": "reads per second", + "refId": "A", + "step": 8, + "target": "" + }, + { + "expr": "sum by (instance) (irate(node_disk_writes_completed_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "writes per second", + "refId": "B", + "step": 8 + }, + { + "expr": "sum by (instance) (irate(node_disk_io_time_seconds_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "io time", + "refId": "C", + "step": 8 + }, + { + "expr": "sum by (instance) (irate(node_disk_reads_completed_total{instance=~\"$server\"}[5m])) + sum by (instance) (irate(node_disk_writes_completed_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "IOPS", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IOPs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 10, + "x": 10, + "y": 13 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "written", + "yaxis": 1 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(node_disk_read_bytes_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "read", + "metric": "node_disk_bytes_read", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(node_disk_written_bytes_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "written", + "metric": "node_disk_bytes_written", + "refId": "B", + "step": 2 + }, + { + "expr": "sum(irate(node_disk_io_time_seconds_total{instance=~\"$server\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "io time", + "metric": "node_disk_io_time_ms", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "I/O Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 13 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(node_filesystem_free_bytes{fstype=~\"xfs|ext4\",instance=~\"$server\"} / node_filesystem_size_bytes{fstype=~\"xfs|ext4\",instance=~\"$server\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40, + "target": "" + } + ], + "thresholds": "0.10, 0.25", + "title": "Free Filesystem Space (Lowest)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$server\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Load 1m", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_free_bytes{fstype=~\"xfs|ext4\",instance=~\"$server\"} - 100", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk free in GB", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "11.3.3.13:9100", + "value": "11.3.3.13:9100" + }, + "datasource": "prom-fed", + "definition": "label_values(node_boot_time_seconds,instance)", + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "server", + "options": [], + "query": "label_values(node_boot_time_seconds,instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Host Details", + "uid": "000000004" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-api-server.json b/deploy/grafana/dashboards/General/kubernetes-api-server.json new file mode 100644 index 000000000..bfb360f03 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-api-server.json @@ -0,0 +1,1126 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"apiserver\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(apiserver_request_total{job=\"apiserver\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(apiserver_request_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (verb, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"apiserver\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "etcd_helper_cache_entry_total{job=\"apiserver\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ETCD Cache Entry Total", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(etcd_helper_cache_hit_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (intance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} hit", + "refId": "A" + }, + { + "expr": "sum(rate(etcd_helper_cache_miss_total{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} miss", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ETCD Cache Hit/Miss Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_get_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} get", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99,sum(rate(etcd_request_cache_add_duration_seconds_bucket{job=\"apiserver\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} miss", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ETCD Cache Duration 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"apiserver\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"apiserver\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"apiserver\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(apiserver_request_total{job=\"apiserver\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / API server", + "uid": "09ec8aa1e996d6ffcd6817bbaff4db1b" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-cluster.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-cluster.json new file mode 100644 index 000000000..36d41e742 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-cluster.json @@ -0,0 +1,2274 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - avg(rate(node_cpu_seconds_total{mode=\"idle\", cluster=\"$cluster\"}[1m]))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_cpu_cores{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 - sum(:node_memory_MemAvailable_bytes:sum{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Requests Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "format": "percentunit", + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) / sum(kube_node_status_allocatable_memory_bytes{cluster=\"$cluster\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Memory Limits Commitment", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\"}) by (namespace) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Pods", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Workloads", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": true, + "linkTooltip": "Drill down to workloads", + "linkUrl": "./d/a87fb0d919ec0ea5f6543124e16c42a5/k8s-resources-workloads-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell_1", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "count(avg(mixin_pod_workload{cluster=\"$cluster\"}) by (workload, namespace)) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", container!=\"\"}) by (namespace) / sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\"}) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Requests by Namespace", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Requests", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/85a562078cdf77779eaa1add43ccec1e/k8s-resources-namespace?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Namespace: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\".+\"}[$interval])) by (namespace)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{namespace}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(node_cpu_seconds_total, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Cluster", + "uid": "efa86fd1d0c121a26444b636a3f509a8" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-pods.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-pods.json new file mode 100644 index 000000000..9d54ac0ca --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-pods.json @@ -0,0 +1,1630 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Pods)", + "uid": "85a562078cdf77779eaa1add43ccec1e" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-workloads.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-workloads.json new file mode 100644 index 000000000..075a330cd --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-namespace-workloads.json @@ -0,0 +1,1853 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}} - {{workload_type}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Running Pods", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$__cell_2", + "pattern": "workload", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}) by (workload, workload_type)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n* on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload_type=\"$type\"}\n) by (workload, workload_type)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down to pods", + "linkUrl": "./d/a164a7f0339f99e89cea5cb47e9be617/k8s-resources-workload?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-workload=$__cell&var-type=$type", + "pattern": "workload", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Workload Type", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "workload_type", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Workload: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{workload}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [], + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Namespace (Workloads)", + "uid": "a87fb0d919ec0ea5f6543124e16c42a5" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-node-pods.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-node-pods.json new file mode 100644 index 000000000..956ce409c --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-node-pods.json @@ -0,0 +1,817 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", node=\"$node\"}) by (pod) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\", container!=\"\"}) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage (w/o cache)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_requests_memory_bytes{node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_working_set_bytes{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod) / sum(kube_pod_container_resource_limits_memory_bytes{node=\"$node\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_rss{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_cache{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_memory_swap{cluster=\"$cluster\", node=\"$node\",container!=\"\"}) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "node", + "multi": false, + "name": "node", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, node)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Node (Pods)", + "uid": "200ac8fdbfbb74b39aff88118e4d1c2c" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-pod.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-pod.json new file mode 100644 index 000000000..e005b8bc3 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-pod.json @@ -0,0 +1,1417 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", cluster=\"$cluster\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (RSS)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (Cache)", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{container}} (Swap)", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Usage (RSS)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Cache)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Usage (Swap)", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Container", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "container", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"POD\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}) by (container) / sum(kube_pod_container_resource_requests_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container!=\"\"}) by (container) / sum(kube_pod_container_resource_limits_memory_bytes{namespace=\"$namespace\", pod=\"$pod\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(container_memory_rss{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sum(container_memory_cache{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sum(container_memory_swap{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container != \"\", container != \"POD\"}) by (container)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval])) by (pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Pod", + "uid": "6581e46e4e5c7ba40a07646395ef7b23" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-compute-resources-workload.json b/deploy/grafana/dashboards/General/kubernetes-compute-resources-workload.json new file mode 100644 index 000000000..5ec57f315 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-compute-resources-workload.json @@ -0,0 +1,1783 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "CPU Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "CPU Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "CPU Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_cpu_cores{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Usage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Memory Usage", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Requests %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Memory Limits", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "bytes" + }, + { + "alias": "Memory Limits %", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_requests_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(\n container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container!=\"\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n/sum(\n kube_pod_container_resource_limits_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\"}\n * on(namespace,pod)\n group_left(workload, workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\", workload_type=\"$type\"}\n) by (pod)\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Quota", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory Quota", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Current Receive Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Transmit Bandwidth", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "./d/6581e46e4e5c7ba40a07646395ef7b23/k8s-resources-pod?var-datasource=$datasource&var-cluster=$cluster&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Current Network Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(avg(irate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Container Bandwidth by Pod: Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_receive_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_network_transmit_packets_dropped_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$interval])\n* on (namespace,pod) \ngroup_left(workload,workload_type) mixin_pod_workload{cluster=\"$cluster\", namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "workload", + "multi": false, + "name": "workload", + "options": [], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "type", + "multi": false, + "name": "type", + "options": [], + "query": "label_values(mixin_pod_workload{cluster=\"$cluster\", namespace=\"$namespace\", workload=\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Compute Resources / Workload", + "uid": "a164a7f0339f99e89cea5cb47e9be617" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-controller-manager.json b/deploy/grafana/dashboards/General/kubernetes-controller-manager.json new file mode 100644 index 000000000..db739fead --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-controller-manager.json @@ -0,0 +1,980 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-controller-manager\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_adds_total{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Add Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(workqueue_depth{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Depth", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 5, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\"}[5m])) by (instance, name, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Work Queue Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-controller-manager\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-controller-manager\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kube-controller-manager\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"kube-controller-manager\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kube-controller-manager\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(process_cpu_seconds_total{job=\"kube-controller-manager\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Controller Manager", + "uid": "72e0e05bef5099e5f049b05fdc429ed4" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-kubelet.json b/deploy/grafana/dashboards/General/kubernetes-kubelet.json new file mode 100644 index 000000000..394497103 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-kubelet.json @@ -0,0 +1,2210 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{cluster=\"$cluster\", job=\"kubelet\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_pod_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Pods", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(kubelet_running_container_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Running Container", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\", state=\"actual_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Actual Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(volume_manager_total_volumes{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\",state=\"desired_state_of_world\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Volume Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(kubelet_node_config_error{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Config Error Count", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (operation_type, instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_runtime_operations_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 10, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_runtime_operations_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Operation duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 11, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" + }, + { + "expr": "sum(rate(kubelet_pod_worker_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 12, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_start_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} pod", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} worker", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Pod Start Duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 13, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_duration_seconds_count{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 14, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(storage_operation_errors_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Error Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 15, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "hideEmpty": "true", + "hideZero": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": true, + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(storage_operation_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_name, volume_plugin, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_name}} {{volume_plugin}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Storage Operation Duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 16, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_cgroup_manager_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{operation_type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager operation rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 17, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_cgroup_manager_duration_seconds_bucket{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, operation_type, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{operation_type}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cgroup manager 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "Pod lifecycle event generator", + "fill": 1, + "gridPos": {}, + "id": 18, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubelet_pleg_relist_duration_seconds_count{cluster=\"$cluster\", job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 19, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_interval_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist interval", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 20, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "PLEG relist duration", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 21, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "RPC Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 22, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{cluster=\"$cluster\",job=\"kubelet\", instance=~\"$instance\"}[5m])) by (instance, verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request duration 99th quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 23, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 24, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 25, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\",job=\"kubelet\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(kubelet_runtime_operations{cluster=\"$cluster\", job=\"kubelet\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Kubelet", + "uid": "3138fa155d5915769fbded898ac09fd9" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-networking-cluster.json b/deploy/grafana/dashboards/General/kubernetes-networking-cluster.json new file mode 100644 index 000000000..0b4dac96e --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-networking-cluster.json @@ -0,0 +1,1247 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "columns": [ + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Namespace", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&refresh=30s&var-namespace=$__cell", + "pattern": "namespace", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "title": "Current Status", + "transform": "table", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 6, + "panels": [ + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 7, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 8, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 9, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth History", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 15, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 24, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 24, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".+\"}[$interval:$resolution])) by (namespace))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{namespace}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "30s", + "rows": [], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Networking / Cluster", + "uid": "ff635a025bcfea7bc3dd4f508990a3e9" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-networking-namespace-pods.json b/deploy/grafana/dashboards/General/kubernetes-networking-namespace-pods.json new file mode 100644 index 000000000..8cc27c496 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-networking-namespace-pods.json @@ -0,0 +1,1271 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace", + "unit": "Bps" + }, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [ + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "100%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bandwidth Received", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bandwidth Transmitted", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Pod", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/7a18067ce943a40ae25454675c19ff5c/kubernetes-networking-pod?orgId=1&refresh=30s&var-namespace=$namespace&var-pod=$__cell", + "pattern": "pod", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + } + ], + "title": "Current Status", + "transform": "table", + "type": "table" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 12, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "30s", + "rows": [], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Networking / Namespace (Pods)", + "uid": "8b7a8b326d7a6f1f04244066368c67af" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-networking-namespace-workload.json b/deploy/grafana/dashboards/General/kubernetes-networking-namespace-workload.json new file mode 100644 index 000000000..817864bee --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-networking-namespace-workload.json @@ -0,0 +1,1303 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "columns": [ + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + }, + { + "text": "", + "value": "" + } + ], + "datasource": "$datasource", + "fill": 1, + "fontSize": "90%", + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "lines": true, + "linewidth": 1, + "minSpan": 24, + "nullPointMode": "null as zero", + "renderer": "flot", + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": false + }, + "spaceLength": 10, + "span": 24, + "styles": [ + { + "alias": "Time", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Current Bandwidth Received", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Current Bandwidth Transmitted", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Received", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Average Bandwidth Transmitted", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Rate of Received Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #F", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Received Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Rate of Transmitted Packets Dropped", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #H", + "thresholds": [], + "type": "number", + "unit": "pps" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Drill down", + "linkUrl": "d/728bf77cc1166d2f3133bf25846876cc/kubernetes-networking-workload?orgId=1&refresh=30s&var-namespace=$namespace&var-type=$type&var-workload=$__cell", + "pattern": "workload", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "C", + "step": 10 + }, + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "D", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "E", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "F", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "G", + "step": 10 + }, + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "H", + "step": 10 + } + ], + "title": "Current Status", + "transform": "table", + "type": "table" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 6, + "panels": [ + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 7, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 8, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 9, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 12, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 15, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\", workload_type=\"$type\"}) by (workload))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{workload}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "30s", + "rows": [], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [], + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\".+\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Networking / Namespace (Workload)", + "uid": "bbb2a765a623ae38130206c7d94a160f" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-networking-pod.json b/deploy/grafana/dashboards/General/kubernetes-networking-pod.json new file mode 100644 index 000000000..8dc58247b --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-networking-pod.json @@ -0,0 +1,1067 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "height": 9, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Received", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "decimals": 0, + "format": "time_series", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "height": 9, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 12, + "nullPointMode": "connected", + "nullText": null, + "options": { + "fieldOptions": { + "calcs": [ + "last" + ], + "defaults": { + "max": 10000000000, + "min": 0, + "title": "$namespace: $pod", + "unit": "Bps" + }, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "dark-green", + "index": 0, + "value": null + }, + { + "color": "dark-yellow", + "index": 1, + "value": 5000000000 + }, + { + "color": "dark-red", + "index": 2, + "value": 7000000000 + } + ], + "values": false + } + }, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 12, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution]))", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Current Rate of Bytes Transmitted", + "type": "gauge", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 8, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\", pod=~\"$pod\"}[$interval:$resolution])) by (pod)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "30s", + "rows": [], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(container_network_receive_packets_total{namespace=~\"$namespace\"}, pod)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Networking / Pod", + "uid": "7a18067ce943a40ae25454675c19ff5c" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-networking-workload.json b/deploy/grafana/dashboards/General/kubernetes-networking-workload.json new file mode 100644 index 000000000..9e78c09f0 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-networking-workload.json @@ -0,0 +1,1045 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Current Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Current Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 5, + "panels": [ + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 6, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Received", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "aliasColors": {}, + "breakpoint": "50%", + "cacheTimeout": null, + "combine": { + "label": "Others", + "threshold": 0 + }, + "datasource": "$datasource", + "fontSize": "80%", + "format": "Bps", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 7, + "interval": null, + "legend": { + "percentage": true, + "percentageDecimals": null, + "show": true, + "values": true + }, + "legendType": "Right side", + "maxDataPoints": 3, + "nullPointMode": "connected", + "pieType": "donut", + "targets": [ + { + "expr": "sort_desc(avg(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "instant": null, + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "title": "Average Rate of Bytes Transmitted", + "type": "grafana-piechart-panel", + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Average Bandwidth", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 8, + "panels": [], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Bandwidth HIstory", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Receive Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_bytes_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Transmit Bandwidth", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Packets", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": true, + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 14, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_receive_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Received Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 2, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "minSpan": 12, + "nullPointMode": "connected", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum(irate(container_network_transmit_packets_dropped_total{namespace=~\"$namespace\"}[$interval:$resolution])\n* on (namespace,pod)\ngroup_left(workload,workload_type) mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\", workload_type=\"$type\"}) by (pod))\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{pod}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rate of Transmitted Packets Dropped", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Errors", + "titleSize": "h6", + "type": "row" + } + ], + "refresh": "30s", + "rows": [], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "kube-system", + "value": "kube-system" + }, + "datasource": "$datasource", + "definition": "label_values(container_network_receive_packets_total, namespace)", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(container_network_receive_packets_total, namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "", + "value": "" + }, + "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\"}, workload)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "workload", + "options": [], + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\"}, workload)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "deployment", + "value": "deployment" + }, + "datasource": "$datasource", + "definition": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "type", + "options": [], + "query": "label_values(mixin_pod_workload{namespace=~\"$namespace\", workload=~\"$workload\"}, workload_type)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "resolution", + "options": [ + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + } + ], + "query": "30s,5m,1h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "5m", + "value": "5m" + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "4h", + "value": "4h" + } + ], + "query": "4h", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "interval", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Networking / Workload", + "uid": "728bf77cc1166d2f3133bf25846876cc" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-persistent-volumes.json b/deploy/grafana/dashboards/General/kubernetes-persistent-volumes.json new file mode 100644 index 000000000..5d17cd1a0 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-persistent-volumes.json @@ -0,0 +1,488 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Persistent Volumes", + "uid": "919b92a8e8041bd567af9edab12c840c" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-pods.json b/deploy/grafana/dashboards/General/kubernetes-pods.json new file mode 100644 index 000000000..b7732ac2c --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-pods.json @@ -0,0 +1,575 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "$datasource", + "enable": true, + "expr": "time() == BOOL timestamp(rate(kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[2m]) > 0)", + "hide": false, + "iconColor": "rgba(215, 44, 44, 1)", + "name": "Restarts", + "showIn": 0, + "tags": [ + "restart" + ], + "type": "rows" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(container) (container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current: {{ container }}", + "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{ container }}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"memory\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{ container }}", + "refId": "C" + }, + { + "expr": "sum by(container) (container_memory_cache{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", container=~\"$container\", container!=\"POD\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cache: {{ container }}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (container) (irate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", image!=\"\", pod=\"$pod\", container=~\"$container\", container!=\"POD\"}[4m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current: {{ container }}", + "refId": "A" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_requests{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Requested: {{ container }}", + "refId": "B" + }, + { + "expr": "sum by(container) (kube_pod_container_resource_limits{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", resource=\"cpu\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Limit: {{ container }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sort_desc(sum by (pod) (irate(container_network_receive_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RX: {{ pod }}", + "refId": "A" + }, + { + "expr": "sort_desc(sum by (pod) (irate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}[4m])))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TX: {{ pod }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by (container) (kube_pod_container_status_restarts_total{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\", container=~\"$container\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Restarts: {{ container }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Total Restarts Per Container", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_pod_info, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Pod", + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(kube_pod_info{cluster=\"$cluster\", namespace=~\"$namespace\"}, pod)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "Container", + "multi": false, + "name": "container", + "options": [], + "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=\"$pod\"}, container)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Pods", + "uid": "ab4f13a9892a76a4d21ce8c2445bf4ea" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-proxy.json b/deploy/grafana/dashboards/General/kubernetes-proxy.json new file mode 100644 index 000000000..67c905862 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-proxy.json @@ -0,0 +1,1046 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-proxy\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_sync_proxy_rules_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rules Sync Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99,rate(kubeproxy_sync_proxy_rules_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rule Sync Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(kubeproxy_network_programming_duration_seconds_count{job=\"kube-proxy\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rate", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Programming Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-proxy\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-proxy\",instance=~\"$instance\",verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-proxy\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kube-proxy\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"kube-proxy\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kube-proxy\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(kubeproxy_network_programming_duration_seconds_bucket{job=\"kube-proxy\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Proxy", + "uid": "632e265de029684c40b21cb76bca4f94" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-scheduler.json b/deploy/grafana/dashboards/General/kubernetes-scheduler.json new file mode 100644 index 000000000..3a8269326 --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-scheduler.json @@ -0,0 +1,917 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=\"kube-scheduler\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Up", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "min" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(scheduler_e2e_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} e2e", + "refId": "A" + }, + { + "expr": "sum(rate(scheduler_binding_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} binding", + "refId": "B" + }, + { + "expr": "sum(rate(scheduler_scheduling_algorithm_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "sum(rate(scheduler_volume_scheduling_duration_seconds_count{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} volume", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 5, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_e2e_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} e2e", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_binding_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} binding", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_scheduling_algorithm_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} scheduling algorithm", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(scheduler_volume_scheduling_duration_seconds_bucket{job=\"kube-scheduler\",instance=~\"$instance\"}[5m])) by (instance, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} volume", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scheduling latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"2..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "2xx", + "refId": "A" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"3..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "3xx", + "refId": "B" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"4..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "4xx", + "refId": "C" + }, + { + "expr": "sum(rate(rest_client_requests_total{job=\"kube-scheduler\", instance=~\"$instance\",code=~\"5..\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5xx", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Kube API Request Rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 8, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"POST\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Post Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": "true", + "avg": false, + "current": "true", + "max": false, + "min": false, + "rightSide": "true", + "show": "true", + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{job=\"kube-scheduler\", instance=~\"$instance\", verb=\"GET\"}[5m])) by (verb, url, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{verb}} {{url}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Get Request Latency 99th Quantile", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes{job=\"kube-scheduler\", instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total{job=\"kube-scheduler\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"kube-scheduler\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(process_cpu_seconds_total{job=\"kube-scheduler\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / Scheduler", + "uid": "2e6b6a3b4bddf1427b3a55aa1311c656" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/kubernetes-statefulsets.json b/deploy/grafana/dashboards/General/kubernetes-statefulsets.json new file mode 100644 index 000000000..4f3e504cc --- /dev/null +++ b/deploy/grafana/dashboards/General/kubernetes-statefulsets.json @@ -0,0 +1,833 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "cores", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "CPU", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "GB", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}) / 1024^3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Memory", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "Bps", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 4, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(container_network_transmit_bytes_total{job=\"kubelet\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$statefulset.*\"}[3m])) + sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\",pod=~\"$statefulset.*\"}[3m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Network", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "height": "100px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Desired Replicas", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Replicas of current version", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_status_observed_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"$statefulset\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Observed Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "$datasource", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Metadata Generation", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(kube_statefulset_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas specified", + "refId": "A" + }, + { + "expr": "max(kube_statefulset_status_replicas{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas created", + "refId": "B" + }, + { + "expr": "min(kube_statefulset_status_replicas_ready{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ready", + "refId": "C" + }, + { + "expr": "min(kube_statefulset_status_replicas_current{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "replicas of current version", + "refId": "D" + }, + { + "expr": "min(kube_statefulset_status_replicas_updated{job=\"kube-state-metrics\", statefulset=\"$statefulset\", cluster=\"$cluster\", namespace=\"$namespace\"}) without (instance, pod)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "updated", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Replicas", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(kube_statefulset_metadata_generation, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Name", + "multi": false, + "name": "statefulset", + "options": [], + "query": "label_values(kube_statefulset_metadata_generation{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, statefulset)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Kubernetes / StatefulSets", + "uid": "a31c1f46e6f727cb37c0d731a7245005" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/load-metrics.json b/deploy/grafana/dashboards/General/load-metrics.json new file mode 100644 index 000000000..244372a72 --- /dev/null +++ b/deploy/grafana/dashboards/General/load-metrics.json @@ -0,0 +1,523 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(host) * on (instance) group_left(nodename) node_load1", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{host}}", + "refId": "A", + "step": 2 + }, + { + "expr": "(node_load1) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load avg on VM's", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((node_filesystem_size_bytes{ mountpoint=\"/\", job=\"vm-node-exporter\"} - node_filesystem_free_bytes{ mountpoint=\"/\"}) * 100\n/ \nnode_filesystem_size_bytes{mountpoint=\"/\"} * on (instance) group_left(nodename) node_uname_info )\nby (nodename)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "A", + "step": 2 + }, + { + "expr": "sum((node_filesystem_size_bytes{ mountpoint=\"/\", job=\"vm-node-exporter\"} - node_filesystem_free_bytes{ mountpoint=\"/\"}) * 100 / node_filesystem_size_bytes{mountpoint=\"/\"} * on (instance) group_left(nodename) node_uname_info ) by (nodename)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk utilization %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes - node_memory_MemFree_bytes - node_memory_Cached_bytes - node_memory_Buffers_bytes - node_memory_Slab_bytes) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{device!=\"lo\"}[5m]) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "In: {{nodename}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network_IN Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_transmit_bytes_total{device!=\"lo\"}[5m]) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network_out Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Load metrics", + "uid": "000000007" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/lp-pipeline-metrics.json b/deploy/grafana/dashboards/General/lp-pipeline-metrics.json new file mode 100644 index 000000000..696186a11 --- /dev/null +++ b/deploy/grafana/dashboards/General/lp-pipeline-metrics.json @@ -0,0 +1,3404 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1560770475550, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Publish Pipeline Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Composite search index Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit History Indexer Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Image Tagging Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit Event Generator Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Publish Pipeline Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 3 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Composite search index Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 3 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit History Indexer Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 3 + }, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Image Tagging Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit Event Generator Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Publish Pipeline Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 6 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Composite search indexer Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 6 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Audit History Indexer Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 6 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "", + "title": "Image Tagging Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "", + "title": "Audit Event Generator Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 16, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "audit-history-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit History Indexer Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 17, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "query": "SELECT \"error\", \"failed\" FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "audit-history-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit History Indexer Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 18, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "audit-event-generator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit Event Generator Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 19, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "audit-event-generator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit Event Generator Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 20, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "publish-pipeline" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Publish Pipeline Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 21, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "publish-pipeline" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Publish Pipeline Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 22, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "composite-search-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Composite Indexer Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 23, + "interval": "> 300s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "query": "SELECT \"error\", \"failed\" FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "composite-search-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Composite Indexer Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"consumer_lag\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'composite-search-indexer' AND \"env\" =~ /^$env$/) AND $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Composite Search - Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 29, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"consumer_lag\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'publish-pipeline' AND \"env\" =~/^$env$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Publish Pipeline - Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 31, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"consumer_lag\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'audit-event-generator' AND \"env\" =~ /^$env$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit Event Generator - Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"consumer_lag\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'audit-history-indexer' AND \"env\" =~ /^$env$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Audit History Indexer - Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"consumer_lag\") FROM \"pipeline_metrics\" WHERE (\"job_name\" = 'asset-enrichment' AND \"env\" =~ /^$env$/) AND $timeFilter GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Asset Enrichment - Lag", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pipeline" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "ntpprod", + "value": "ntpprod" + }, + "datasource": "Monitoring Events", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "tagValuesQuery": "env", + "tagsQuery": "pipeline_metrics", + "type": "custom", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "LP Pipeline Metrics", + "uid": "9I50utWZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/lpa-metrics.json b/deploy/grafana/dashboards/General/lpa-metrics.json new file mode 100644 index 000000000..1f9d965eb --- /dev/null +++ b/deploy/grafana/dashboards/General/lpa-metrics.json @@ -0,0 +1,830 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1558372555439, + "links": [], + "panels": [ + { + "columns": [], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "interval": "> 3600s", + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "dateFormat": "MMMM D, YYYY LT", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Time spent", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Input", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Output", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + } + ], + "hide": false, + "measurement": "analytics_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "query": "SELECT sum(\"number_of_output\") AS \"Output\", sum(\"success\") AS \"Success\", sum(\"time_spent\") AS \"Time spent\" FROM \"autogen\".\"analytics_metrics\" WHERE (\"env\" =~ /^$env$/) AND $timeFilter GROUP BY time(1d)", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "number_of_output" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Output" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Success" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "time_spent" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "Time spent" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + } + ] + } + ], + "title": "Data Products Daily Aggregated Metrics", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "1d" + ], + "type": "time" + } + ], + "measurement": "analytics_metrics", + "policy": "default", + "query": "SELECT count(\"job_name\") FROM \"autogen\".\"analytics_metrics\" WHERE \"env\" =~ /^$env$/ AND $timeFilter AND success = 1 group by time(1d)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + }, + { + "params": [ + "success" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Data Products Daily Success Rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "analytics_metrics.count": "#BF1B00", + "analytics_metrics.failure": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + " 1d " + ], + "type": "time" + } + ], + "measurement": "analytics_metrics", + "policy": "default", + "query": "SELECT count(\"job_name\") FROM \"autogen\".\"analytics_metrics\" WHERE \"env\" =~ /^$env$/ AND $timeFilter AND success = 1 group by time(1d)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + }, + { + "params": [ + "failure" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "is_success", + "operator": "=", + "value": "false" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Data Products Daily Failure Rate", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 4, + "interval": "> 3600s", + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "dateFormat": "MMMM D, YYYY LT", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "Time spent", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Input", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Output", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "job_name" + ], + "type": "tag" + } + ], + "hide": false, + "measurement": "analytics_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "query": "SELECT \"number_of_input\" AS \"Input\", \"number_of_output\" AS \"Output\", \"success\" AS \"Success\", \"time_spent\" AS \"Time spent\" FROM \"autogen\".\"analytics_metrics\" WHERE (\"env\" =~ /^$env$/) AND $timeFilter GROUP BY \"job_name\"", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "number_of_input" + ], + "type": "field" + }, + { + "params": [ + "Input" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "number_of_output" + ], + "type": "field" + }, + { + "params": [ + "Output" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [ + "Success" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "time_spent" + ], + "type": "field" + }, + { + "params": [ + "Time spent" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + } + ] + } + ], + "title": "# Data Products All jobs Daily Metrics", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fontSize": "100%", + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 5, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "job_id" + ], + "type": "tag" + } + ], + "measurement": "analytics_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "query": "SELECT \"number_of_input\" AS \"Inputs\", \"number_of_output\" AS \"Outputs\", \"success\" AS \"Success\", \"time_spent\" AS \"TimeSpent\", \"tag\" AS \"ClientKey\" FROM \"autogen\".\"analytics_metrics\" WHERE (\"env\" =~ /^$env$/ AND \"job_name\" = 'DataExhaustJob' AND \"job_id\" <> 'AnalyticsDataPipeline') AND $timeFilter GROUP BY \"job_id\"", + "rawQuery": true, + "refId": "A", + "resultFormat": "table", + "select": [ + [ + { + "params": [ + "number_of_input" + ], + "type": "field" + }, + { + "params": [ + "Inputs" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "number_of_output" + ], + "type": "field" + }, + { + "params": [ + "Outputs" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "success" + ], + "type": "field" + }, + { + "params": [ + "Success" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "time_spent" + ], + "type": "field" + }, + { + "params": [ + "TimeSpent" + ], + "type": "alias" + } + ], + [ + { + "params": [ + "tag" + ], + "type": "field" + }, + { + "params": [ + "ClientKey" + ], + "type": "alias" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "DataExhaustJob" + }, + { + "condition": "AND", + "key": "job_id", + "operator": "<>", + "value": "AnalyticsDataPipeline" + } + ] + } + ], + "title": "DataExhaust Request Details", + "transform": "table", + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "pipeline" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "ntpprod", + "value": "ntpprod" + }, + "datasource": "Monitoring Events", + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "refresh": 0, + "regex": "", + "skipUrlSync": false, + "tagValuesQuery": "env", + "tagsQuery": "pipeline_metrics", + "type": "custom", + "useTags": false + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "LPA Metrics", + "uid": "IKtAutZZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/master-dashboard-ds1.json b/deploy/grafana/dashboards/General/master-dashboard-ds1.json new file mode 100644 index 000000000..47db1ff95 --- /dev/null +++ b/deploy/grafana/dashboards/General/master-dashboard-ds1.json @@ -0,0 +1,2796 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1558376126363, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 32, + "panels": [], + "repeat": null, + "title": "Pipeline Stats", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Publish Pipeline Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 1 + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Composite search index Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit History Indexer Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 1 + }, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Image Tagging Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit Event Generator Errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Publish Pipeline Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 4 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Composite search index Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 4 + }, + "id": 23, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit History Indexer Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 4 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Image Tagging Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 25, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit Event Generator Failure", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 26, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Publish Pipeline Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 7 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Composite search indexer Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 7 + }, + "id": 28, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "", + "title": "Audit History Indexer Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 7 + }, + "id": 29, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "", + "title": "Image Tagging Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 30, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "", + "title": "Audit Event Generator Success", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 33, + "panels": [], + "repeat": null, + "title": "Daily Peak of Request per Second", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_request_status_count[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "All", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Daily Peaks of Request per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 34, + "panels": [], + "repeat": null, + "title": "Max % HTTP 4xx Errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|5..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max % HTTP 4xx Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|4..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max % HTTP 5xx Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 35, + "panels": [], + "repeat": null, + "title": "API Requests Error %", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{status_code!~\"2..|3..|5..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "4xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{status_code!~\"2..|3..|4..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "5xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 36, + "panels": [], + "repeat": null, + "title": "Load Avg on VMs", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(host) * on (instance) group_left(nodename) node_load1", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{host}}", + "refId": "A", + "step": 120 + }, + { + "expr": "(node_load1) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load avg on VMs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 37, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Usage of Containers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 38, + "panels": [], + "repeat": null, + "title": "Top 10 API Requests", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(kong_request_count[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "metric": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 API Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 39, + "panels": [], + "repeat": null, + "title": "APIs Avg/Min/Max Latency", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_latency_time", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "APIs Average/Min/Max Response Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "ntpprod", + "value": "ntpprod" + }, + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus_DS1", + "definition": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now/d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Master Dashboard DS1", + "uid": "3O5mepZWz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/nginx-new.json b/deploy/grafana/dashboards/General/nginx-new.json new file mode 100644 index 000000000..285cb8d84 --- /dev/null +++ b/deploy/grafana/dashboards/General/nginx-new.json @@ -0,0 +1,331 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": false, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_http_requests_total[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "All", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Traffic : Requests / Sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_http_requests_total{status=~\"2..|3..\"}[5m])) by (status) / ignoring(status) group_left sum(increase(nginx_http_requests_total[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Success %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_http_requests_total{status!~\"2..|3..\"}[5m])) by (status) / ignoring(status) group_left sum(increase(nginx_http_requests_total[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{status}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Nginx-New", + "uid": "S_H3_RNWz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/nginx.json b/deploy/grafana/dashboards/General/nginx.json new file mode 100644 index 000000000..b97add21f --- /dev/null +++ b/deploy/grafana/dashboards/General/nginx.json @@ -0,0 +1,331 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": false, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_request_status_count[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "All", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Traffic : Requests / Sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code=~\"2..|3..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Success %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Nginx", + "uid": "000000005" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/nginx_backup.json b/deploy/grafana/dashboards/General/nginx_backup.json new file mode 100644 index 000000000..8b955c8db --- /dev/null +++ b/deploy/grafana/dashboards/General/nginx_backup.json @@ -0,0 +1,330 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": false, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_request_status_count[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "All", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Traffic : Requests / Sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code=~\"2..|3..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Success %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Nginx_backup", + "uid": "VS4QN-GZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/noc-dashboard-ds1.json b/deploy/grafana/dashboards/General/noc-dashboard-ds1.json new file mode 100644 index 000000000..28d85e257 --- /dev/null +++ b/deploy/grafana/dashboards/General/noc-dashboard-ds1.json @@ -0,0 +1,1611 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 0 + }, + "height": "50px", + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Publish Pipeline Errors", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 0 + }, + "height": "50px", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Composite search index Errors", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 10, + "y": 0 + }, + "height": "50px", + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit History Indexer Errors", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 0 + }, + "height": "50px", + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Image Tagging Errors", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 0 + }, + "height": "50px", + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"error\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 20, + "tags": [] + } + ], + "thresholds": "1,1", + "title": "Audit Event Generator Errors", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "height": "50px", + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Publish Pipeline Failure", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 4, + "y": 3 + }, + "height": "50px", + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Composite search index Failure", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 10, + "y": 3 + }, + "height": "50px", + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'audit-history-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit History Indexer Failure", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 3 + }, + "height": "50px", + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'image-tagging' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Image Tagging Failure", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(218, 109, 18, 0.89)", + "rgba(211, 12, 12, 0.9)" + ], + "datasource": "InfluxDB_Datapipeline", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 3 + }, + "height": "50px", + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "dsType": "influxdb", + "expr": "SELECT sum(\"success\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'publish-pipeline' AND $timeFilter", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"failed\") FROM \"pipeline_metrics\" WHERE \"env\" =~ /^ntpprod$/ AND \"job_name\" = 'audit-event-generator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "step": 40, + "tags": [] + } + ], + "thresholds": "1:1", + "title": "Audit Event Generator Failure", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|5..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "4xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|4..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "5xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum((node_filesystem_size{ mountpoint=\"/\", job=\"node-exporter\"} - node_filesystem_free{ mountpoint=\"/\"}) * 100/ node_filesystem_size{mountpoint=\"/\"} * on (instance) group_left(host) host ) by (host)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{host}}", + "refId": "A", + "step": 10 + }, + { + "expr": "sum((node_filesystem_size{ mountpoint=\"/\", job=\"vm-node-exporter\"} - node_filesystem_free{ mountpoint=\"/\"}) * 100/ node_filesystem_size{mountpoint=\"/\"} * on (instance) group_left(nodename) node_uname_info ) by (nodename)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk utilization %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(host) * on (instance) group_left(nodename) node_load1", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{host}}", + "refId": "A", + "step": 10 + }, + { + "expr": "(node_load1) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load avg on VM's", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NOC Dashboard DS1", + "uid": "8Lo0upZZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/node-exporter-full.json b/deploy/grafana/dashboards/General/node-exporter-full.json new file mode 100644 index 000000000..c317c6b52 --- /dev/null +++ b/deploy/grafana/dashboards/General/node-exporter-full.json @@ -0,0 +1,19987 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": 1860, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": 151, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Busy state of all CPU cores together", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 1, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "(((count(count(node_cpu_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}) by (cpu))) - avg(sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=~\"$node:$port\",job=~\"$job\"}[5m])))) * 100) / count(count(node_cpu_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}) by (cpu))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 900 + } + ], + "thresholds": "85,95", + "title": "CPU Busy", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": 0, + "description": "Non available RAM memory", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "hideTimeOverride": false, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "((node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_MemFree_bytes{instance=~\"$node:$port\",job=~\"$job\"}) / (node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} )) * 100", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "A", + "step": 900 + }, + { + "expr": "100 - ((node_memory_MemAvailable_bytes{instance=~\"$node:$port\",job=~\"$job\"} * 100) / node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "B", + "step": 900 + } + ], + "thresholds": "80,90", + "title": "Used RAM Memory", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Used Swap", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "((node_memory_SwapTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_SwapFree_bytes{instance=~\"$node:$port\",job=~\"$job\"}) / (node_memory_SwapTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} )) * 100", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "10,25", + "title": "Used SWAP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Used Root FS", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 154, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node:$port\",job=~\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=~\"$node:$port\",job=~\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "80,90", + "title": "Used Root FS", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Busy state of all CPU cores together (1 min average)", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 1, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(node_load1{instance=~\"$node:$port\",job=~\"$job\"}) / count(count(node_cpu_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}) by (cpu)) * 100", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "85, 95", + "title": "CPU System Load (1m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Busy state of all CPU cores together (5 min average)", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 155, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 1, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(node_load5{instance=~\"$node:$port\",job=~\"$job\"}) / count(count(node_cpu_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}) by (cpu)) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "85, 95", + "title": "CPU System Load (5m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Basic CPU / Mem / Disk Gauge", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 46, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "description": "Total number of CPU cores", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(count(node_cpu_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}) by (cpu))", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "", + "title": "CPU Cores", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 2, + "description": "Total RAM", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 75, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "70%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "", + "title": "Total RAM", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 2, + "description": "Total SWAP", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "70%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_memory_SwapTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "", + "title": "Total SWAP", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "decimals": null, + "description": "Total RootFS", + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 23, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 2, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=~\"$node:$port\",job=~\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "70,90", + "title": "Total RootFS", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 2, + "description": "System Load (1m avg)", + "format": "short", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 1, + "nullPointMode": "null", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_load1{instance=~\"$node:$port\",job=~\"$job\"}", + "hide": false, + "intervalFactor": 1, + "refId": "A", + "step": 900 + } + ], + "thresholds": "", + "title": "System Load (1m avg)", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 1, + "description": "System uptime", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "hideTimeOverride": true, + "id": 15, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "null", + "nullText": null, + "postfix": "s", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "node_time_seconds{instance=~\"$node:$port\",job=~\"$job\"} - node_boot_time_seconds{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Uptime", + "transparent": false, + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Basic CPU / Mem / Disk Info", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "275", + "panels": [ + { + "aliasColors": { + "Busy": "#EAB839", + "Busy Iowait": "#890F02", + "Busy other": "#1F78C1", + "Idle": "#052B51", + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "description": "Basic CPU info", + "fill": 4, + "id": 77, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": null, + "sortDesc": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Busy Iowait", + "color": "#890F02" + }, + { + "alias": "Idle", + "color": "#7EB26D" + }, + { + "alias": "Busy System", + "color": "#EAB839" + }, + { + "alias": "Busy User", + "color": "#0A437C" + }, + { + "alias": "Busy Other", + "color": "#6D1F62" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (instance)(rate(node_cpu_seconds_total{mode=\"system\",instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Busy System", + "refId": "B", + "step": 240 + }, + { + "expr": "sum by (instance)(rate(node_cpu_seconds_total{mode='user',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Busy User", + "refId": "D", + "step": 240 + }, + { + "expr": "sum by (instance)(rate(node_cpu_seconds_total{mode='iowait',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy Iowait", + "refId": "E", + "step": 240 + }, + { + "expr": "sum by (instance)(rate(node_cpu_seconds_total{mode=~\".*irq\",instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy IRQs", + "refId": "F", + "step": 240 + }, + { + "expr": "sum (rate(node_cpu_seconds_total{mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Busy Other", + "refId": "A", + "step": 240 + }, + { + "expr": "sum by (mode)(rate(node_cpu_seconds_total{mode='idle',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Idle", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "SWAP Used": "#BF1B00", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap Used": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "description": "Basic memory usage", + "fill": 4, + "id": 78, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "RAM Total", + "color": "#E0F9D7", + "fill": 0, + "stack": false + }, + { + "alias": "RAM Cache + Buffer", + "color": "#052B51" + }, + { + "alias": "RAM Free", + "color": "#7EB26D" + }, + { + "alias": "Avaliable", + "color": "#DEDAF7", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Total", + "refId": "A", + "step": 240 + }, + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_MemFree_bytes{instance=~\"$node:$port\",job=~\"$job\"} - (node_memory_Cached_bytes{instance=~\"$node:$port\",job=~\"$job\"} + node_memory_Buffers_bytes{instance=~\"$node:$port\",job=~\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RAM Used", + "refId": "D", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=~\"$node:$port\",job=~\"$job\"} + node_memory_Buffers_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Cache + Buffer", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAM Free", + "refId": "F", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_SwapFree_bytes{instance=~\"$node:$port\",job=~\"$job\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SWAP Used", + "refId": "G", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Basic CPU / Mem Graph", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "275", + "panels": [ + { + "aliasColors": { + "Recv_bytes_eth2": "#7EB26D", + "Recv_bytes_lo": "#0A50A1", + "Recv_drop_eth2": "#6ED0E0", + "Recv_drop_lo": "#E0F9D7", + "Recv_errs_eth2": "#BF1B00", + "Recv_errs_lo": "#CCA300", + "Trans_bytes_eth2": "#7EB26D", + "Trans_bytes_lo": "#0A50A1", + "Trans_drop_eth2": "#6ED0E0", + "Trans_drop_lo": "#E0F9D7", + "Trans_errs_eth2": "#BF1B00", + "Trans_errs_lo": "#CCA300", + "recv_bytes_lo": "#0A50A1", + "recv_drop_eth0": "#99440A", + "recv_drop_lo": "#967302", + "recv_errs_eth0": "#BF1B00", + "recv_errs_lo": "#890F02", + "trans_bytes_eth0": "#7EB26D", + "trans_bytes_lo": "#0A50A1", + "trans_drop_eth0": "#99440A", + "trans_drop_lo": "#967302", + "trans_errs_eth0": "#BF1B00", + "trans_errs_lo": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "Basic network info per interface", + "fill": 4, + "id": 74, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "recv {{device}}", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(node_network_transmit_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trans {{device}} ", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "pps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 3, + "description": "Disk space used of all filesystems mounted", + "fill": 4, + "height": "", + "id": 152, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'} * 100) / node_filesystem_size_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used Basic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Basic Net / Disk Info", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 470, + "panels": [ + { + "aliasColors": { + "Idle - Waiting for something to happen": "#052B51", + "guest": "#9AC48A", + "idle": "#052B51", + "iowait": "#EAB839", + "irq": "#BF1B00", + "nice": "#C15C17", + "softirq": "#E24D42", + "steal": "#FCE2DE", + "system": "#508642", + "user": "#5195CE" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "description": "", + "fill": 4, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 250, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode=\"system\",instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "interval": "10s", + "intervalFactor": 2, + "legendFormat": "System - Processes executing in kernel mode", + "refId": "A", + "step": 120 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='user',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "User - Normal processes executing in user mode", + "refId": "B", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='nice',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Nice - Niced processes executing in user mode", + "refId": "C", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='idle',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Idle - Waiting for something to happen", + "refId": "F", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='iowait',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Iowait - Waiting for I/O to complete", + "refId": "D", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='irq',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Irq - Servicing interrupts", + "refId": "G", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='softirq',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Softirq - Servicing softirqs", + "refId": "H", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='steal',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "refId": "E", + "step": 240 + }, + { + "expr": "sum by (mode)(irate(node_cpu_seconds_total{mode='guest',instance=~\"$node:$port\",job=~\"$job\"}[5m])) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", + "refId": "I", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Percentage", + "logBase": 1, + "max": "100", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap - Swap memory usage": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839", + "Unused - Free memory unasigned": "#052B51" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "description": "", + "fill": 4, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Harware Corrupted - *./", + "stack": false + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_MemFree_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_Buffers_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_Cached_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_Slab_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_PageTables_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_SwapCached_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Apps - Memory used by user-space applications", + "refId": "Q", + "step": 240 + }, + { + "expr": "node_memory_PageTables_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "refId": "G", + "step": 240 + }, + { + "expr": "node_memory_SwapCached_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "refId": "F", + "step": 240 + }, + { + "expr": "node_memory_Slab_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "refId": "E", + "step": 240 + }, + { + "expr": "node_memory_Cached_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Cache - Parked file data (file content) cache", + "refId": "C", + "step": 240 + }, + { + "expr": "node_memory_Buffers_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "refId": "B", + "step": 240 + }, + { + "expr": "node_memory_MemFree_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Unused - Free memory unasigned", + "refId": "D", + "step": 240 + }, + { + "expr": "(node_memory_SwapTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"} - node_memory_SwapFree_bytes{instance=~\"$node:$port\",job=~\"$job\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Swap - Swap space used", + "refId": "I", + "step": 240 + }, + { + "expr": "node_memory_HardwareCorrupted_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "refId": "O", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Stack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 4, + "id": 84, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "O", + "step": 240 + }, + { + "expr": "irate(node_network_transmit_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "P", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 3, + "description": "", + "fill": 4, + "height": "", + "id": 156, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_size_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 229, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=~\"$node:$port\",job=~\"$job\",device=~\"[a-z]*[a-z]\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 480 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=~\"$node:$port\",job=~\"$job\",device=~\"[a-z]*[a-z]\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOps", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 3, + "description": "", + "fill": 4, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*read*./", + "transform": "negative-Y" + }, + { + "alias": "/.*sda.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=~\"$node:$port\",job=~\"$job\",device=~\"[a-z]*[a-z]\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully read bytes", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=~\"$node:$port\",job=~\"$job\",device=~\"[a-z]*[a-z]\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Successfully written bytes", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "I/O Usage Read / Write", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "io time": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 3, + "description": "", + "fill": 4, + "id": 127, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node:$port\",job=~\"$job\",device=~\"[a-z]*[a-z]\"} [5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Milliseconds spent doing I/Os", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "I/O Usage Times", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": false, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "ms", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "CPU Memory Net Disk", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 136, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", + "refId": "K", + "step": 4 + }, + { + "expr": "node_memory_Active_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", + "refId": "J", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Active / Inactive", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 135, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Committed_AS - *./" + }, + { + "alias": "/.*CommitLimit - *./", + "color": "#BF1B00", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Committed_AS_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_CommitLimit_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", + "refId": "M", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Commited", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 191, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Inactive_file_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Inactive_anon_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Inactive_anon - Anonymous and swap cache on inactive LRU list, including tmpfs (shmem)", + "refId": "D", + "step": 4 + }, + { + "expr": "node_memory_Active_file_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_file - File-backed memory on active LRU list", + "refId": "B", + "step": 4 + }, + { + "expr": "node_memory_Active_anon_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Active_anon - Anonymous and swap cache on active least-recently-used (LRU) list, including tmpfs", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Active / Inactive Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 130, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Writeback_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writeback - Memory which is actively being written back to disk", + "refId": "J", + "step": 4 + }, + { + "expr": "node_memory_WritebackTmp_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", + "refId": "K", + "step": 4 + }, + { + "expr": "node_memory_Dirty_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Writeback and Dirty", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 138, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Mapped_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Mapped - Used memory in mapped pages files which have been mmaped, such as libraries", + "refId": "A", + "step": 4 + }, + { + "expr": "node_memory_Shmem_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Shared and Mapped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 131, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_SUnreclaim_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SUnreclaim - Part of Slab, that cannot be reclaimed on memory pressure", + "refId": "O", + "step": 4 + }, + { + "expr": "node_memory_SReclaimable_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SReclaimable - Part of Slab, that might be reclaimed, such as caches", + "refId": "N", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Slab", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_VmallocChunk_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocChunk - Largest contigious block of vmalloc area which is free", + "refId": "H", + "step": 4 + }, + { + "expr": "node_memory_VmallocTotal_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocTotal - Total size of vmalloc memory area", + "refId": "I", + "step": 4 + }, + { + "expr": "node_memory_VmallocUsed_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "VmallocUsed - Amount of vmalloc area which is used", + "refId": "O", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Vmalloc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 159, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Bounce_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bounce - Memory used for block device bounce buffers", + "refId": "N", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Bounce", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 129, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Inactive *./", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_AnonHugePages_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonHugePages - Memory in anonymous huge pages", + "refId": "D", + "step": 4 + }, + { + "expr": "node_memory_AnonPages_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AnonPages - Memory in user pages not backed by files", + "refId": "G", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Anonymous", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 160, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_KernelStack_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", + "refId": "N", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Kernel", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 140, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Free{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Free - Huge pages in the pool that are not yet allocated", + "refId": "I", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Rsvd{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Rsvd - Huge pages for which a commitment to allocate from the pool has been made, but no allocation has yet been made", + "refId": "J", + "step": 4 + }, + { + "expr": "node_memory_HugePages_Surp{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages_Surp - Huge pages in the pool above the value in /proc/sys/vm/nr_hugepages", + "refId": "K", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory HugePages Counter", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#806EB7", + "Total RAM + Swap": "#806EB7", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_HugePages_Total{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "HugePages - Total size of the pool of huge pages", + "refId": "L", + "step": 4 + }, + { + "expr": "node_memory_Hugepagesize_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Hugepagesize - Huge Page size", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory HugePages Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 128, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_DirectMap1G{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DirectMap1G - Amount of pages mapped as this size", + "refId": "J", + "step": 4 + }, + { + "expr": "node_memory_DirectMap2M_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap2M - Amount of pages mapped as this size", + "refId": "K", + "step": 4 + }, + { + "expr": "node_memory_DirectMap4k_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "DirectMap4K - Amount of pages mapped as this size", + "refId": "L", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory DirectMap", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 137, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_Unevictable_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable - Amount of unevictable memory that can't be swapped out for a variety of reasons", + "refId": "P", + "step": 4 + }, + { + "expr": "node_memory_Mlocked_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "MLocked - Size of pages locked to memory using the mlock() system call", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Unevictable and MLocked", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 132, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_NFS_Unstable_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet commited to the storage", + "refId": "L", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory NFS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Memory Detail Meminfo", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 176, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgpgin{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesin - Page in operations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgpgout{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pagesout - Page out operations", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Pages In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*out/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pswpin{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpin - Pages swapped in", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pswpout{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pswpout - Pages swapped out", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Pages Swap In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 197, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgdeactivate{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgdeactivate - Pages moved from active to inactive", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgfree{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgfree - Page free operations", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgactivate{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgactivate - Pages moved from inactive to active", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 175, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Pgfault - Page major and minor fault operations", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgfault{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgfault - Page major and minor fault operations", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgmajfault{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgmajfault - Major page fault operations", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgfault{instance=~\"$node:$port\",job=~\"$job\"}[5m]) - irate(node_vmstat_pgmajfault{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgminfault - Minor page fault operations", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Faults", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Faults", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 172, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_kswapd_inodesteal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Kswapd_inodesteal - Pages reclaimed via kswapd inode freeing", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pginodesteal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgindesteal - Pages reclaimed via inode freeing", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Pages Reclaimed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 184, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pageoutrun{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pageoutrun - Kswapd calls to page reclaim", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_allocstall{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Allocstall - Direct reclaim calls", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_zone_reclaim_failed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Zone_reclaim_failed - Zone reclaim failures", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Calls Reclaimed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Calls", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 200, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgrotated{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgrotated - Pages rotated to tail of the LRU", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Rotate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 170, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_drop_pagecache{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Drop_pagecache - Calls to drop page cache pages", + "refId": "N", + "step": 4 + }, + { + "expr": "node_vmstat_drop_slab{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Drop_slab - Calls to drop slab cache pages", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Drop", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Calls", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 183, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_slabs_scanned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Slabs_scanned - Slab pages scanned", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Scan Slab", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 181, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_unevictable_pgs_cleared{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_cleared - Unevictable pages cleared", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_culled{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_culled - Unevictable pages culled", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_mlocked{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_mlocked - Unevictable pages mlocked", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_munlocked{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_munlocked - Unevictable pages munlocked", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_rescued{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_rescued- Unevictable pages rescued", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_scanned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable_pgs_scanned - Unevictable pages scanned", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_vmstat_unevictable_pgs_stranded{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unevictable_pgs_stranded - Unevictable pages stranded", + "refId": "H", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Unevictable Pages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 174, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgalloc_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgalloc_dma - Dma mem page allocations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgalloc_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgalloc_dma32 - Dma32 mem page allocations", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgalloc_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgalloc_movable - Movable mem page allocations", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgalloc_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgalloc_normal - Normal mem page allocations", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Allocation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 177, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgrefill_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgrefill_dma - Dma mem pages inspected in refill_inactive_zone", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgrefill_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgrefill_dma32 - Dma32 mem pages inspected in refill_inactive_zone", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgrefill_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgrefill_movable - Movable mem pages inspected in refill_inactive_zone", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgrefill_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgrefill_normal - Normal mem pages inspected in refill_inactive_zone", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Refill", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 179, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgsteal_direct_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_direct_dma - Dma mem pages stealed", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_direct_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_direct_dma32 - Dma32 mem pages scanned", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_direct_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_direct_movable - Movable mem pages scanned", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_direct_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_direct_normal - Normal mem pages scanned", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Steal Direct", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 198, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgsteal_kswapd_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_kswapd_dma - Dma mem pages scanned by kswapd", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_kswapd_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_kswapd_dma32 - Dma32 mem pages scanned by kswapd", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_kswapd_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_kswapd_movable - Movable mem pages scanned by kswapd", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgsteal_kswapd_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgsteal_kswapd_normal - Normal mem pages scanned by kswapd", + "refId": "I", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Steal Kswapd", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 192, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgscan_direct_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_direct_dma - Dma mem pages scanned", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_direct_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_direct_dma32 - Dma32 mem pages scanned", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_direct_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_direct_movable - Movable mem pages scanned", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_direct_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_direct_normal - Normal mem pages scanned", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_direct_throttle{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "Pgscan_direct_throttle - ", + "refId": "F", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Scan Direct", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 178, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_pgscan_kswapd_dma{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_kswapd_dma - Dma mem pages scanned by kswapd", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_kswapd_dma32{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_kswapd_dma32 - Dma32 mem pages scanned by kswapd", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_kswapd_movable{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_kswapd_movable - Movable mem pages scanned by kswapd", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgscan_kswapd_normal{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgscan_kswapd_normal - Normal mem pages scanned by kswapd", + "refId": "I", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Scan Kswapd", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 169, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*fail*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_compact_free_scanned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_free_scanned - Pages scanned for freeing by compaction daemon", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_compact_isolated{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_isolated - Page isolations for memory compaction", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_compact_migrate_scanned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_migrate_scanned - Pages scanned for migration by compaction daemon", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Compact", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Active": "#99440A", + "Buffers": "#58140C", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Dirty": "#6ED0E0", + "Free": "#B7DBAB", + "Inactive": "#EA6460", + "Mapped": "#052B51", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "Slab_Cache": "#EAB839", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Total": "#511749", + "Total RAM": "#052B51", + "Total RAM + Swap": "#052B51", + "Total Swap": "#614D93", + "VmallocUsed": "#EA6460" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 189, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*fail*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_compact_fail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_fail - Unsuccessful compactions for high order allocations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_compact_stall{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_stall - Failures to even start compacting", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_vmstat_compact_success{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Compact_sucess - Successful compactions for high order allocations", + "refId": "F", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Compactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Compactions", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 190, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_kswapd_high_wmark_hit_quickly{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Kswapd_high_wmark_hit_quickly - Times high watermark reached quickly", + "refId": "N", + "step": 4 + }, + { + "expr": "node_vmstat_kswapd_low_wmark_hit_quickly{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Kswapd_low_wmark_hit_quickly - Times low watermark reached quickly", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Kswapd Watermark", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 171, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_htlb_buddy_alloc_fail{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Htlb_buddy_alloc_fail - Huge TLB page buddy allocation failures", + "refId": "N", + "step": 4 + }, + { + "expr": "node_vmstat_htlb_buddy_alloc_success{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Htlb_buddy_alloc_success - Huge TLB page buddy allocation successes", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Buddy Alloc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Allocations", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 173, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_numa_foreign{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_foreign - Foreign NUMA zone allocations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_hit{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_hit - Successful allocations from preferred NUMA zone", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_interleave{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_interleave - Interleaved NUMA allocations in each zone for each NUMA node", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_local{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_local - Successful allocations from local NUMA zone", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_miss{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_miss - Unsuccessful allocations from preferred NUMA zona", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_other{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_other - Unsuccessful allocations from local NUMA zone", + "refId": "I", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Numa Allocations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Allocations", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 193, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Numa_pages_migrated - *./", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_numa_pages_migrated{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_pages_migrated - NUMA page migrations", + "refId": "J", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgmigrate_fail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgmigrate_fail - Unsuccessful NUMA page migrations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_pgmigrate_success{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pgmigrate_success - Successful NUMA page migrations", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Numa Page Migrations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 194, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_numa_hint_faults{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_hint_faults - NUMA hint faults trapped", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_hint_faults_local{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_hint_faults_local - Hinting faults to local nodes", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Numa Hints", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "HInts", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 196, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 6, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_numa_pte_updates{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Numa_pte_updates - NUMA page table entry updates", + "refId": "K", + "step": 4 + }, + { + "expr": "irate(node_vmstat_numa_huge_pte_updates{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Numa_huge_pte_updates - NUMA huge page table entry updates", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Numa Table Updates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Updates", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 199, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_thp_split{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Thp_split - Transparent huge page splits", + "refId": "F", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory THP Splits", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Splits", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 182, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_workingset_activate{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workingset_activate - Page activations to form the working set", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_workingset_nodereclaim{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workingset_nodereclaim - NUMA node working set page reclaims", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_vmstat_workingset_refault{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workingset_refault - Refaults of previously evicted pages", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Workingset", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 180, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_vmstat_thp_collapse_alloc{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Thp_collapse_alloc - Transparent huge page collapse allocations", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_vmstat_thp_collapse_alloc_failed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Thp_collapse_alloc_failed - Transparent huge page collapse allocation failures", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_vmstat_thp_zero_page_alloc{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Thp_zero_page_alloc - Transparent huge page zeroed page allocations", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_vmstat_thp_zero_page_alloc_failed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Thp_zero_page_alloc_failed - Transparent huge page zeroed page allocation failures", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_vmstat_thp_fault_alloc{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Thp_fault_alloc - Transparent huge page fault allocations", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_vmstat_thp_fault_fallback{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "Thp_fault_fallback - Transparent huge page fault fallbacks", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory THP Allocations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Allocations", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Memory Detail Vmstat", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 185, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_active_anon{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Active_anon - Active anonymous memory pages", + "refId": "B", + "step": 240 + }, + { + "expr": "node_vmstat_nr_active_file{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Active_file - Active file memory memory pages", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Active", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 228, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_inactive_anon{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Inactive_anon - Inactive anonymous memory pages in each zone for each NUMA node", + "refId": "A", + "step": 240 + }, + { + "expr": "node_vmstat_nr_inactive_file{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Inactive_file - Inactive file memory pages in each zone for each NUMA node", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Inactive", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 188, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_slab_reclaimable{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reclaimable - Instantaneous reclaimable slab pages", + "refId": "A", + "step": 240 + }, + { + "expr": "node_vmstat_nr_slab_unreclaimable{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unreclaimable - Instantaneous unreclaimable slab pages", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Reclaimed / Unreclaimed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 186, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_free_pages{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Free_pages - Free pages", + "refId": "B", + "step": 240 + }, + { + "expr": "node_vmstat_nr_written{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Written - Pages written out in each zone for each NUMA node", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Free / Written", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 218, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_dirty{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty - Pages in dirty state", + "refId": "C", + "step": 240 + }, + { + "expr": "node_vmstat_nr_bounce{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Bounce - Bounce buffer pages", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Dirty / Bounce", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 201, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_unevictable{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unevictable - Unevictable pages", + "refId": "B", + "step": 240 + }, + { + "expr": "node_vmstat_nr_mlock{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Mlock - Pages under mlock", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Pages Unevictable / Mlock", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 214, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_shmem{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Shmem - Shared memory pages", + "refId": "H", + "step": 240 + }, + { + "expr": "node_vmstat_nr_mapped{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Mapped - Mapped pagecache pages in each zone for each NUMA node", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Shmem / Mapped", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 212, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_kernel_stack{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Kernel_stack - Pages of kernel stack", + "refId": "F", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Kernel_stack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 203, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_writeback{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writeback - Writeback pages", + "refId": "G", + "step": 240 + }, + { + "expr": "node_vmstat_nr_writeback_temp{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Writeback_temp - Temporary writeback pages", + "refId": "H", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Pages Writeback", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 205, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_file_pages{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "File_pages - File pagecache pages in each zone for each NUMA node", + "refId": "F", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page File_pages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 206, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_dirty_background_threshold{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty_background_threshold - Background writeback threshold", + "refId": "D", + "step": 240 + }, + { + "expr": "node_vmstat_nr_dirty_threshold{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Dirty_threshold - Dirty throttling threshold", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Dirty Threshold", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 208, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_unstable{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unstable - Pages unstable state in each zone for each NUMA node", + "refId": "D", + "step": 240 + }, + { + "expr": "node_vmstat_nr_dirtied{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Dirtied - Pages entering dirty state in each zone for each NUMA node", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Unstable / Dirtied", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 209, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_page_table_pages{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Page_table_pages - Page table pages in each zone for each NUMA node", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Page_table_pages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 217, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_alloc_batch{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc_batch - Pages allocated to other zones due to insufficient memory for each zone for each NUMA node", + "refId": "D", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Alloc_batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 213, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_isolated_anon{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Isolated_anon - Isolated anonymous memory pages in each zone for each NUMA node", + "refId": "D", + "step": 240 + }, + { + "expr": "node_vmstat_nr_isolated_file{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Isolated_file - Isolated file memory pages in each zone for each NUMA node", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Isolated", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 216, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_anon_pages{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anon_pages - Anonymous mapped pagecache pages in each zone for each NUMA node", + "refId": "E", + "step": 240 + }, + { + "expr": "node_vmstat_nr_anon_transparent_hugepages{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anon_transparent_hugepages - Anonymous transparent huge pages in each zone for each NUMA node", + "refId": "F", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Anon", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "Apps": "#629E51", + "Buffers": "#614D93", + "Cache": "#6D1F62", + "Cached": "#511749", + "Committed": "#508642", + "Free": "#0A437C", + "Harware Corrupted - Amount of RAM that the kernel identified as corrupted / not working": "#CFFAFF", + "Inactive": "#584477", + "PageTables": "#0A50A1", + "Page_Tables": "#0A50A1", + "RAM_Free": "#E0F9D7", + "Slab": "#806EB7", + "Slab_Cache": "#E0752D", + "Swap": "#BF1B00", + "Swap_Cache": "#C15C17", + "Swap_Free": "#2F575E", + "Unused": "#EAB839" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 2, + "fill": 2, + "id": 204, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_vmstat_nr_free_cma{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Free_cma - Free Contiguous Memory Allocator pages in each zone for each NUMA node", + "refId": "G", + "step": 240 + }, + { + "expr": "node_vmstat_nr_vmscan_write{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Vmscan_write - Pages written by VM scanner from LRU", + "refId": "B", + "step": 240 + }, + { + "expr": "node_vmstat_nr_vmscan_immediate_reclaim{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 2, + "legendFormat": "Immediate_reclaim - Prioritise for reclaim when writeback ends in each zone for each NUMA node", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Page Misc", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Pages", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Memory Detail Vmstat Counters", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_context_switches_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Context switches", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_intr_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Interrupts", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Context Switches / Interrupts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 1m", + "refId": "A", + "step": 480 + }, + { + "expr": "node_load5{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 5m", + "refId": "B", + "step": 480 + }, + { + "expr": "node_load15{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Load 15m", + "refId": "C", + "step": 480 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "System Load", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Load", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 259, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_interrupts_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ type }} - {{ info }}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Interrupts Detail", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 64, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Max*./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_max_fds{instance=~\"$node:$port\",job=~\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum open file descriptors", + "refId": "A", + "step": 240 + }, + { + "expr": "process_open_fds{instance=~\"$node:$port\",job=~\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open file descriptors", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Descriptors", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 151, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_entropy_available_bits{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Entropy available to random number generators", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Entropy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Entropy", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 62, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_procs_blocked{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes blocked waiting for I/O to complete", + "refId": "A", + "step": 240 + }, + { + "expr": "node_procs_running{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Processes in runnable state", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Processes State", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Processes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 148, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_forks_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Processes forks second", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Processes Forks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Forks / sec", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 149, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes virtual memory size in bytes", + "refId": "C", + "step": 240 + }, + { + "expr": "process_resident_memory_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Processes resident memory size in bytes", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Processes Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 168, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_sync_status{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Is clock synchronized to a reliable server (1 = yes, 0 = no)", + "refId": "B", + "step": 240 + }, + { + "expr": "node_timex_frequency_adjustment_ratio{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Local clock frequency adjustment", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time Syncronized Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 260, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Variation*./", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_timex_estimated_error_seconds{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Estimated error in seconds", + "refId": "B", + "step": 240 + }, + { + "expr": "node_timex_offset_seconds{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Time offset in between local system and reference clock", + "refId": "A", + "step": 240 + }, + { + "expr": "node_timex_maxerror_seconds{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Maximum error in seconds", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time Syncronized Drift", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 158, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Critical*./", + "color": "#E24D42", + "fill": 0 + }, + { + "alias": "/.*Max*./", + "color": "#EF843C", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_hwmon_temp_celsius{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} temp", + "refId": "A", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Alarm", + "refId": "B", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_celsius{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical", + "refId": "C", + "step": 240 + }, + { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Critical Historical", + "refId": "D", + "step": 240 + }, + { + "expr": "node_hwmon_temp_max_celsius{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ chip }} {{ sensor }} Max", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Hardware temperature monitor", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "celsius", + "label": "Temperature", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "System Detail", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_completed_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - Reads completed", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_writes_completed_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Writes completed", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOps Completed", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "IO read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{device}} - Read bytes", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_written_bytes_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Written bytes", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk R/W Data", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 3, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "hide": false, + "intervalFactor": 4, + "legendFormat": "{{device}} - Read time ms", + "refId": "A", + "step": 8 + }, + { + "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Write time ms", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk R/W Time", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Millisec. read (-) / write (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time weighted", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOs Weighted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 133, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Read.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_reads_merged_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Read merged", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_disk_writes_merged_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "{{device}} - Write merged", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk R/W Merged", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "I/Os", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 3, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO time ms", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Milliseconds Spent Doing I/Os", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#BA43A9" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_disk_io_now{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 4, + "legendFormat": "{{device}} - IO now", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IOs Current in Progress", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "iops", + "label": "I/Os", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*sda_.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*sdb_.*/", + "color": "#EAB839" + }, + { + "alias": "/.*sdc_.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*sdd_.*/", + "color": "#EF843C" + }, + { + "alias": "/.*sde_.*/", + "color": "#E24D42" + }, + { + "alias": "/.*sda1.*/", + "color": "#584477" + }, + { + "alias": "/.*sda2_.*/", + "color": "#B7DBAB" + }, + { + "alias": "/.*sda3_.*/", + "color": "#F4D598" + }, + { + "alias": "/.*sdb1.*/", + "color": "#0A50A1" + }, + { + "alias": "/.*sdb2.*/", + "color": "#BF1B00" + }, + { + "alias": "/.*sdb3.*/", + "color": "#E0752D" + }, + { + "alias": "/.*sdc1.*/", + "color": "#962D82" + }, + { + "alias": "/.*sdc2.*/", + "color": "#614D93" + }, + { + "alias": "/.*sdc3.*/", + "color": "#9AC48A" + }, + { + "alias": "/.*sdd1.*/", + "color": "#65C5DB" + }, + { + "alias": "/.*sdd2.*/", + "color": "#F9934E" + }, + { + "alias": "/.*sdd3.*/", + "color": "#EA6460" + }, + { + "alias": "/.*sde1.*/", + "color": "#E0F9D7" + }, + { + "alias": "/.*sdd2.*/", + "color": "#FCEACA" + }, + { + "alias": "/.*sde3.*/", + "color": "#F9E2D2" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_textfile_scrape_error{instance=~\"$node:$port\",job=~\"$job\"}", + "intervalFactor": 4, + "legendFormat": "Textfile scrape error (1 = true)", + "refId": "A", + "step": 8 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Open Error File", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Errors", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Disk Detail", + "titleSize": "h6" + }, + { + "collapse": true, + "height": 391, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 3, + "description": "", + "fill": 2, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_avail_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Available", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "node_filesystem_free_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free", + "refId": "B", + "step": 2 + }, + { + "expr": "node_filesystem_size_bytes{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Size", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem space available", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 41, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files_free{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - Free file nodes", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Nodes Free", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "File Nodes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filefd_maximum{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "Max open files", + "refId": "A", + "step": 8 + }, + { + "expr": "node_filefd_allocated{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open files", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Descriptor", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Files", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 219, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_files{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - File nodes total", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "File Nodes Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "File Nodes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { + "/ ReadOnly": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 2, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_filesystem_readonly{instance=~\"$node:$port\",job=~\"$job\",device!~'rootfs'}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{mountpoint}} - ReadOnly", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Filesystem in ReadOnly", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Read Only", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Filesystem Detail", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": { + "receive_packets_eth0": "#7EB26D", + "receive_packets_lo": "#E24D42", + "transmit_packets_eth0": "#7EB26D", + "transmit_packets_lo": "#E24D42" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 60, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_packets_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_packets_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit", + "refId": "P", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic by Packets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 142, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.errors.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.errors.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.errors.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.errors.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.errors.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.errors.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.errors.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_errs_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive errors", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_errs_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Rransmit errors", + "refId": "F", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 143, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.drop.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.drop.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.drop.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.drop.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.drop.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.drop.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.drop.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_drop_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive drop", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_drop_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit drop", + "refId": "H", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Drop", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 141, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.compressed.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.compressed.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.compressed.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.compressed.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.compressed.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.compressed.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.compressed.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_compressed_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive compressed", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_compressed_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit compressed", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Compressed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 146, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.multicast.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.multicast.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.multicast.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.multicast.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.multicast.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.multicast.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.multicast.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_multicast_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive multicast", + "refId": "M", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Multicast", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 144, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.fifo.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.fifo.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.fifo.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.fifo.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.fifo.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.fifo.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.fifo.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_fifo_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive fifo", + "refId": "I", + "step": 4 + }, + { + "expr": "irate(node_network_transmit_fifo_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit fifo", + "refId": "J", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Fifo", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 145, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.frame.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.frame.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.frame.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.frame.*/", + "color": "#EF843C" + }, + { + "alias": "/.*eth3.*.frame.*/", + "color": "#E24D42" + }, + { + "alias": "/.*eth4.*.frame.*/", + "color": "#1F78C1" + }, + { + "alias": "/.*eth5.*.frame.*/", + "color": "#BA43A9" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_frame_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}} - Receive frame", + "refId": "K", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Frame", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "label": "Packets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 231, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.carrier.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.carrier.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.carrier.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.carrier.*/", + "color": "#EF843C" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_transmit_carrier_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Statistic transmit_carrier", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Carrier", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 232, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Trans.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*lo.*.carrier.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*eth0.*.carrier.*/", + "color": "#EAB839" + }, + { + "alias": "/.*eth1.*.carrier.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*eth2.*.carrier.*/", + "color": "#EF843C" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_transmit_colls_total{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}} - Transmit colls", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Traffic Colls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "NF conntrack limit", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_nf_conntrack_entries{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack entries", + "refId": "O", + "step": 4 + }, + { + "expr": "node_nf_conntrack_entries_limit{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NF conntrack limit", + "refId": "P", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "NF Contrack", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Entries", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 230, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_arp_entries{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ device }} - ARP entries", + "refId": "O", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ARP Entries", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Entries", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Traffic Detail", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 63, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_alloc{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_alloc - Allocated sockets", + "refId": "D", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_inuse{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "refId": "E", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_mem{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_mem - Used memory for tcp", + "refId": "F", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_orphan{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_orphan - Orphan sockets", + "refId": "H", + "step": 240 + }, + { + "expr": "node_sockstat_TCP_tw{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_tw - Sockets wating close", + "refId": "I", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockstat TCP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Sockets", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 124, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_UDPLITE_inuse{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "refId": "J", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_inuse{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "refId": "K", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "UDP_mem - Used memory for udp", + "refId": "L", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockstat UDP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Sockets", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 126, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_sockets_used{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sockets_used - Sockets currently in use", + "refId": "N", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockstat Used", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Sockets", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 220, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_TCP_mem_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCP_mem_bytes - ", + "refId": "G", + "step": 240 + }, + { + "expr": "node_sockstat_UDP_mem_bytes{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "UDP_mem_bytes - ", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockstat Memory Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Bytes", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "id": 125, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_sockstat_FRAG_inuse{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "FRAG_inuse - Frag sockets currently in use", + "refId": "A", + "step": 240 + }, + { + "expr": "node_sockstat_FRAG_memory{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "FRAG_memory - Used memory for frag", + "refId": "B", + "step": 240 + }, + { + "expr": "node_sockstat_RAW_inuse{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RAW_inuse - Raw sockets currently in use", + "refId": "C", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockstat FRAG / RAW", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Sockets", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Sockstat", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": null, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Discards.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*NoRoutes.*/", + "color": "#EAB839" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Ip_InReceives{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InReceives - IP inreceives", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_DefaultTTL{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "DefaultTTL - Default TTL", + "refId": "B", + "step": 10 + }, + { + "expr": "irate(node_netstat_Ip_InDelivers{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InDelivers - IP indelivers", + "refId": "I", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_OutRequests{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutRequests - IP outrequests", + "refId": "P", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 221, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Octets.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*McastPkts.*/", + "color": "#EAB839" + }, + { + "alias": "/.*McastOctets.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*BcastPkts.*/", + "color": "#EF843C" + }, + { + "alias": "/.*BcastOctets.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InOctets - Received octets", + "refId": "K", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutOctets - Sent octets", + "refId": "Q", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP In / Out Octets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Octects out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 119, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Octets.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*McastPkts.*/", + "color": "#EAB839" + }, + { + "alias": "/.*McastOctets.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*BcastPkts.*/", + "color": "#EF843C" + }, + { + "alias": "/.*BcastOctets.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InBcastPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InBcastPkts - Received IP broadcast datagrams", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutBcastPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutBcastPkts - Sent IP broadcast datagrams", + "refId": "N", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Bcast", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 222, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Octets.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*McastPkts.*/", + "color": "#EAB839" + }, + { + "alias": "/.*McastOctets.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*BcastPkts.*/", + "color": "#EF843C" + }, + { + "alias": "/.*BcastOctets.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InBcastOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InBcastOctets - Received IP broadcast octets", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutBcastOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutBcastOctets - Sent IP broadcast octects", + "refId": "M", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Bcast Octets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Octets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 120, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Octets.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*McastPkts.*/", + "color": "#EAB839" + }, + { + "alias": "/.*McastOctets.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*BcastPkts.*/", + "color": "#EF843C" + }, + { + "alias": "/.*BcastOctets.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InMcastPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InMcastPkts - Received IP multicast datagrams", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutMcastPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutMcastPkts - Sent IP multicast datagrams", + "refId": "P", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Mcast", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 223, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Octets.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*McastPkts.*/", + "color": "#EAB839" + }, + { + "alias": "/.*McastOctets.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*BcastPkts.*/", + "color": "#EF843C" + }, + { + "alias": "/.*BcastOctets.*/", + "color": "#E24D42" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InMcastOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InMcastOctets - Received IP multicast octets", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_OutMcastOctets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutMcastOctets - Sent IP multicast octets", + "refId": "O", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Mcast Octets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Octets out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 81, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": null, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Ip_ForwDatagrams{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ForwDatagrams - IP outforwdatagrams", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_Forwarding{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forwarding - IP forwarding", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Forwarding", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 122, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": null, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Ip_FragCreates{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "FragCreates - IP fragmentation creations", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_FragFails{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "FragFails - IP fragmentation failures", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_FragOKs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "FragOKs - IP fragmentation oks", + "refId": "G", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Fragmented", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 51, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_IpExt_InCEPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InCEPkts - Congestion Experimented datagrams in", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InECT0Pkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InECT0Pkts - Datagrams received with ECT(0)", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InECT1Pkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InECT1Pkt - Datarams received with ECT(1)", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InNoECTPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InNoECTPkts - Datagrams received with NOECT", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP ECT / CEP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 123, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": null, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Ip_ReasmFails{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ReasmFails - IP reassembly failures", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_ReasmOKs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ReasmOKs - IP reassembly oks", + "refId": "R", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_ReasmReqds{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ReasmReqds - IP reassembly requireds", + "refId": "S", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_ReasmTimeout{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ReasmTimeout - IP reasmtimeout", + "refId": "T", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Reasambled", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 118, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": null, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Discards.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*NoRoutes.*/", + "color": "#EAB839" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Ip_InDiscards{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InDiscards - IP indiscards", + "refId": "J", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_InHdrErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InHdrErrors - IP inhdrerrors", + "refId": "K", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_InUnknownProtos{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InUnknownProtos - IP inunknownprotos", + "refId": "M", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_OutDiscards{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutDiscards - IP outdiscards", + "refId": "N", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_OutNoRoutes{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutNoRoutes - IP outnoroutes", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InNoRoutes{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InNoRoutes - IP datagrams discarded due to no routes in forwarding path", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InCsumErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InCsumErrors - IP datagrams with checksum errors", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_IpExt_InTruncatedPkts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InTruncatedPkts - IP datagrams discarded due to frame not carrying enough data", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_Ip_InAddrErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InAddrErrors - IP inaddrerrors", + "refId": "D", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Netstat IP Errors / Discards", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Netstat", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 52, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*InSegs.*/", + "color": "#CCA300" + }, + { + "alias": "/.*OutSegs.*/", + "color": "#CCA300" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Tcp_InCsumErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InCsumErrors - Segments received with checksum errors", + "refId": "E", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_InErrs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "refId": "F", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_InSegs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "G", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_OutRsts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutRsts - Segments sent containing the RST flag", + "refId": "I", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_OutSegs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "refId": "J", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_RetransSegs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Segments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Segments out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 85, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*MaxConn *./", + "color": "#890F02", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_CurrEstab{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "refId": "C", + "step": 240 + }, + { + "expr": "node_netstat_Tcp_MaxConn{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "MaxConn - Limit on the total number of TCP connections the entity can support (Dinamic is \"-1\")", + "refId": "H", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 86, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_netstat_Tcp_RtoAlgorithm{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "RtoAlgorithm - Algorithm used to determine the timeout value used for retransmitting unacknowledged octets", + "refId": "M", + "step": 4 + }, + { + "expr": "node_netstat_Tcp_RtoMax{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RtoMax - Maximum value permitted by a TCP implementation for the retransmission timeout, measured in milliseconds", + "refId": "N", + "step": 240 + }, + { + "expr": "node_netstat_Tcp_RtoMin{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RtoMin - Minimum value permitted by a TCP implementation for the retransmission timeout, measured in milliseconds", + "refId": "O", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Retransmission", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Milliseconds", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 82, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ActiveOpens - TCP connections that have made a direct transition to the SYN-SENT state from the CLOSED state", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_AttemptFails{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "AttemptFails - TCP connections that have made a direct transition to the CLOSED state from either the SYN-SENT and SYN-RCVD", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_EstabResets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "EstabResets - TCP connections that have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state", + "refId": "D", + "step": 240 + }, + { + "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "PassiveOpens - TCP connections that have made a direct transition to the SYN-RCVD state from the LISTEN state", + "refId": "K", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Direct Transition", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Netstat TCP", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 94, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPAbortOnClose{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPAbortOnClose - Connections aborted due to early user close", + "refId": "V", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPAbortOnData{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPAbortOnData - Connections aborted due to unexpected data", + "refId": "W", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPAbortOnLinger{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPAbortOnLinger - Connections aborted in lingered state after being closed", + "refId": "X", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPAbortOnMemory{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPAbortOnMemory - Connections aborted before attached to a socket", + "refId": "Y", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPAbortOnTimeout{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPAbortOnTimeout - Connections aborted due timeout", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPAbortFailed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPAbortFailed - Connections aborted without send RST due insuffient memory", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPTimeouts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPTimeouts - Other TCP connections timeouts", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Aborts / Tiemouts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Connections", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 92, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_DelayedACKLocked{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DelayedACKLocked - Delayed acks further delayed because of locked socket", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_DelayedACKLost{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DelayedACKLost - Times quick ack mode was activated", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_DelayedACKs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DelayedACKs - Delayed acks sent", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Delayed ACK", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 91, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Sent.*/", + "transform": "negative-Y" + }, + { + "alias": "SyncookiesSent - SYN cookies sent", + "color": "#E0F9D7" + }, + { + "alias": "SyncookiesRecv - SYN cookies received", + "color": "#E0F9D7" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_SyncookiesFailed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "refId": "R", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesRecv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "refId": "S", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_SyncookiesSent{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "refId": "T", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSYNChallenge{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SynChallenge - Challenge ACKs sent in response to SYN packets", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPChallengeACK{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "TCPChallengeACK - Challenge ACKs sent (RFC 5961 3.2)", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP SynCookie / Challenge", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 90, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPLossFailures{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPLossFailures - Timeouts in loss state", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPLossProbeRecovery{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPLossProbeRecovery - TCP loss probe recoveries", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPLossProbes{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPLossProbes - Sent TCP loss probes", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPLossUndo{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPLossUndo - Congestion windows recovered without slow start after partial ack", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPLostRetransmit{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPLostRetransmit - Retransmits lost", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP LOSS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 53, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_LockDroppedIcmps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "LockDroppedIcmps - ICMP packets dropped because socket was locked", + "refId": "J", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDeferAcceptDrop{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPDeferAcceptDrop - Dropped ACK frames received by a socket in SYN_RECV state", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPBacklogDrop{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPBacklogDrop - Packets dropped bacause the socket's receive queue was full", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_OutOfWindowIcmps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutOfWindowIcmps - ICMP packets dropped because they were out-of-window", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPMinTTLDrop{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPMinTTLDrop - TCP packets dropped under minTTL condition", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP DROPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 101, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPForwardRetrans{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPForwardRetrans - Packets losts retransmitted with Forward RTO-Recovery", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSlowStartRetrans{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPSlowStartRetrans - Packets losts retransmitted after a slow start", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSpuriousRTOs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSpuriousRTOs - FRTO's successfully detected spurious RTOs", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSpuriousRtxHostQueues{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSpuriousRtxHostQueues - Times detected that the fast clone is not yet freed in tcp_transmit_skb()", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFullUndo{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPFullUndo - Retransmits that undid the CWND reduction", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPRetransFail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPRetransFail - Failed tcp_retransmit_skb() calls", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPPartialUndo{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPPartialUndo - Congestion windows partially recovered using Hoe heuristic", + "refId": "E", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Retrans", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 87, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_PruneCalled{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PruneCalled - Pkts pruned from recv queue because of soc buf overrun", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_RcvPruned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "RcvPruned - Packets pruned from receive queue", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_OfoPruned{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OfoPruned - Packets dropped from out-of-order queue because of socket buffer overrun", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Pruned", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 96, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPDirectCopyFromBacklog{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPDirectCopyFromBacklog - Packets directly received from backlog", + "refId": "M", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDirectCopyFromPrequeue{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPDirectCopyFromPrequeue - Packets directly received from prequeue", + "refId": "N", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Direct Copy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 100, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TW{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TW - TCP sockets finished time wait in fast timer", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TWKilled{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TWKilled - TCP sockets finished time wait in slow timer", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TWRecycled{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TWRecycled - Time wait sockets recycled by time stamp", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPTimeWaitOverflow{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPTimeWaitOverflow - Occurences of time wait bucket overflow", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP TimeWait", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 93, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_PAWSActive{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PAWSActive - Active connections rejected because of time stamp", + "refId": "M", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_PAWSEstab{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PAWSEstab - Pkts rejects in estab connections because of timestamp", + "refId": "N", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_PAWSPassive{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "PAWSPassive - Passive connections rejected because of time stamp", + "refId": "O", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP PAWS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 98, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPSackRecovery{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackRecovery - Times recovered from packet loss by selective acknowledgements", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSackRecoveryFail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackRecoveryFail - Issue while recovering packets lost using selective ACK", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSackShiftFallback{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackShiftFallback - SACKs fallbacks", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSackShifted{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackShifted - SACKs shifted", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSACKDiscard{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackDiscard - Discarded due invalid SACK block.", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSackFailures{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackFailures - Timeouts after SACK recovery", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSackMerged{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSackMerged - SACKs merged", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSACKReneging{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSACKReneging - Bad SACK blocks received", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSACKReorder{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPSACKReorder - Times detected reordering using SACK", + "refId": "I", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP SACK", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 95, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKIgnoredOld{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKIgnoredOld - Discarded packets with duplicate SACK while retransmitting", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKOfoRecv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKOfoRecv - DSACKs for out of order packets received", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKOfoSent{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKOfoSent - DSACKs sent for out of order packets", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKOldSent{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKOldSent - DSACKs sent for old packets", + "refId": "I", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKRecv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKRecv - DSACKs received", + "refId": "J", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKUndo{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPDSACKUndo - Discarded packets with erroneous retransmit", + "refId": "K", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPDSACKIgnoredNoUndo{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPDSACKIgnoredNoUndo - Discarded packets with duplicate SACK", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP DSACK", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 97, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenActive{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenActive - Successful outbound TFO connections", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenActiveFail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenActiveFail - SYN-ACK packets received that did not acknowledge data sent in the SYN packet and caused a retransmissions without SYN data", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenCookieReqd{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenCookieReqd - Inbound SYN packets requesting TFO with TFO set but no cookie", + "refId": "R", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenListenOverflow{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenListenOverflow - Times the fastopen listen queue overflowed", + "refId": "S", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenPassive{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenPassive - Successful inbound TFO connections", + "refId": "T", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastOpenPassiveFail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastOpenPassiveFail - Inbound SYN packets with TFO cookie that was invalid", + "refId": "U", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFastRetrans{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPFastRetrans - Packets lost fast-retransmitted", + "refId": "V", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP FastOpen / FastRetrans", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 99, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPHPAcks{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPHPAcks - Acknowledgments not containing data received", + "refId": "Z", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPHPHits{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPHPHits - Packets header predicted", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPHPHitsToUser{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPHPHitsToUser - Packets header predicted and directly queued to user", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP HP", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 102, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPToZeroWindowAdv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPToZeroWindowAdv - Times window went from zero to non-zero", + "refId": "V", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPWantZeroWindowAdv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPWantZeroWindowAdv - Times zero window announced", + "refId": "W", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPFromZeroWindowAdv{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPFromZeroWindowAdv - Times window went from zero to non-zero", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP ZeroWindow", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 103, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPFACKReorder{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPFACKReorder - Detected packets with re-ordering using FACK", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPTSReorder{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPTSReorder - Times detected packets with re-ordering using timestamp option", + "refId": "S", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Reorder", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 162, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPRenoFailures{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPRenoFailures - Timeouts after reno fast retransmit", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPRenoRecovery{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPRenoRecovery - Times recovered from packet loss due to fast retransmit", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPRenoRecoveryFail{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPRenoRecoveryFail - Times reno fast retransmits failed", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPRenoReorder{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPRenoReorder - Times detected reordering using reno fast retransmit", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Reno", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 163, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPReqQFullDoCookies{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPReqQFullDoCookies - Times SYNCOOKIE was replied to client", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPReqQFullDrop{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPReqQFullDrop - Times SYN request was dropped due to disabled syncookies", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP ReqQ", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 164, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPOFODrop{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPOFODrop - Packets meant to be queued in OFO but dropped because socket rcvbuf limit reached", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPOFOMerge{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPOFOMerge - Packets in OFO that were merged with other packets", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPOFOQueue - Packets queued in OFO queue", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Out of order", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 165, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPMD5NotFound{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPMD5NotFound - Times MD5 hash expected but not found", + "refId": "Z", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPMD5Unexpected{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPMD5Unexpected - Times MD5 hash unexpected but found", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP MD5", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 166, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPPrequeued{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPPrequeued - Packets directly queued to recvmsg prequeue", + "refId": "Z", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPPrequeueDropped{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPPrequeueDropped - Packets dropped from prequeue", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Prequeued", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "description": "", + "fill": 2, + "height": "", + "id": 167, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPRcvCoalesce{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPRcvCoalesce - Times tried to coalesce the receive queue", + "refId": "Z", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPRcvCollapsed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "TCPRcvCollapsed - Packets collapsed in receive queue due to low socket buffer", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Rcv", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 224, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPOrigDataSent{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPOrigDataSent - Outgoing packets with original data", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Original Data", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 225, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_ArpFilter{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ArpFilter - Arp packets filtered", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_IPReversePathFilter{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "IPReversePathFilter - Packets arrive from non directly connected network", + "refId": "G", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Filters", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 226, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPPureAcks{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPPureAcks - Acknowledgments not containing data payload received", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Pure ACK", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 227, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_TCPAutoCorking{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TCPAutoCorking - Times stack detected skb was underused and its flush was deferred", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Auto Corking", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "height": "", + "id": 104, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_TcpExt_BusyPollRxPackets{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "BusyPollRxPackets - Low latency application-fetched packets", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_EmbryonicRsts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "EmbryonicRsts - Resets received for embryonic SYN_RECV sockets", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "refId": "I", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPSchedulerFailed{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "TCPSchedulerFailed - Times receiver scheduled too late for direct processing", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_TcpExt_TCPMemoryPressures{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "intervalFactor": 2, + "legendFormat": "TCPMemoryPressures - TCP ran low on memory", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "TCP Issues", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Netstat TCP Linux MIPs", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 55, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Datagrams.*/", + "color": "#EAB839" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InDatagrams - Datagrams received", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutDatagrams - Datagrams sent", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "UDP In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 109, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Snd.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*bufErrors.*/", + "color": "#70DBED" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Udp_InCsumErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InCsumErrors - Datagrams with checksum errors", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InErrors - Datagrams that could not be delivered to an application", + "refId": "C", + "step": 240 + }, + { + "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RcvbufErrors - Datagrams for which not enough socket buffer memory to receive", + "refId": "F", + "step": 240 + }, + { + "expr": "irate(node_netstat_Udp_SndbufErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SndbufErrors - Datagrams for which not enough socket buffer memory to transmit", + "refId": "G", + "step": 240 + }, + { + "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NoPorts - Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "UDP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 57, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Datagrams.*/", + "color": "#EAB839" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_UdpLite_InDatagrams{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InDatagrams - Packets received", + "refId": "B", + "step": 240 + }, + { + "expr": "irate(node_netstat_UdpLite_OutDatagrams{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutDatagrams - Packets sent", + "refId": "E", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "UDP Lite In / Out", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 2, + "height": "", + "id": 110, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*bufErrors.*/", + "color": "#6ED0E0" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_UdpLite_InCsumErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InCsumErrors - Datagrams with checksum errors", + "refId": "A", + "step": 240 + }, + { + "expr": "irate(node_netstat_UdpLite_InErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InErrors - Datagrams that could not be delivered to an application", + "refId": "C", + "step": 240 + }, + { + "expr": "irate(node_netstat_UdpLite_RcvbufErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "RcvbufErrors - Datagrams for which not enough socket buffer memory to receive", + "refId": "F", + "step": 240 + }, + { + "expr": "irate(node_netstat_UdpLite_SndbufErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "SndbufErrors - Datagrams for which not enough socket buffer memory to transmit", + "refId": "G", + "step": 240 + }, + { + "expr": "irate(node_netstat_UdpLite_NoPorts{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "NoPorts - Datagrams received on a port with no listener", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "UDP Lite Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Datagrams out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Netstat UDP", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 50, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Type3.*/", + "color": "#EAB839" + }, + { + "alias": "/.*SrcQuenchs.*/", + "color": "#705DA0" + }, + { + "alias": "/.*ParmProb.*/", + "color": "#70DBED" + }, + { + "alias": "/.*TimeExcds.*/", + "color": "#EA6460" + }, + { + "alias": "/.*DestUnreachs.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*InErrors.*/", + "color": "#890F02" + }, + { + "alias": "/.*OutErrors.*/", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "refId": "I", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutErrors - Messages which this entity did not send due to problems discovered within ICMP, such as a lack of buffers", + "refId": "V", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InDestUnreachs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InDestUnreachs - Destination Unreachable messages received", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutDestUnreachs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutDestUnreachs - Destination Unreachable messages sent", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(node_netstat_IcmpMsg_InType3{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InType3 - Destination unreachable", + "refId": "K", + "step": 4 + }, + { + "expr": "irate(node_netstat_IcmpMsg_OutType3{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutType3 - Destination unreachable", + "refId": "L", + "step": 4 + }, + { + "expr": "irate(node_netstat_IcmpMsg_InType11{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InType11 - Time Exceeded", + "refId": "C", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP Errors 1", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 147, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Type3.*/", + "color": "#EAB839" + }, + { + "alias": "/.*SrcQuenchs.*/", + "color": "#705DA0" + }, + { + "alias": "/.*ParmProb.*/", + "color": "#70DBED" + }, + { + "alias": "/.*TimeExcds.*/", + "color": "#EA6460" + }, + { + "alias": "/.*DestUnreachs.*/", + "color": "#7EB26D" + }, + { + "alias": "/.*InErrors.*/", + "color": "#890F02" + }, + { + "alias": "/.*OutErrors.*/", + "color": "#890F02" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InCsumErrors{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InCsumErrors - Messages with ICMP checksum errors", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InTimeExcds{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InTimeExcds - Time Exceeded messages received", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutTimeExcds{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutTimeExcds - Time Exceeded messages sent", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InParmProbs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InParmProbs - Parameter Problem messages received", + "refId": "F", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutParmProbs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutParmProb - Parameter Problem messages sent", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InSrcQuenchs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "InSrcQuenchs - Source Quench messages received", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutSrcQuenchs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "OutSrcQuenchs - Source Quench messages sent", + "refId": "J", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP Errors 2", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 115, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*Msgs.*/", + "color": "#6ED0E0" + }, + { + "alias": "/.*Redirects.*/", + "color": "#F9BA8F" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InMsgs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "refId": "J", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InRedirects{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InRedirects - Redirect messages received", + "refId": "L", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutMsgs{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "refId": "W", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutRedirects{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutRedirects - Redirect messages sent. For a host, this object will always be zero, since hosts do not send redirects", + "refId": "Y", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP In / Out - Messages / Redirects", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 112, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*TimestampReps.*/", + "color": "#F9934E" + }, + { + "alias": "/.*Timestamps -.*/", + "color": "#6ED0E0" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InTimestampReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InTimestampReps - Timestamp Reply messages received", + "refId": "O", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InTimestamps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InTimestamps - Timestamp (request) messages received", + "refId": "P", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutTimestampReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutTimestampReps - Timestamp Reply messages sent", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutTimestamps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutTimestamps - Timestamp (request) messages sent", + "refId": "B", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP Timestamps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 114, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*EchoReps.*/", + "color": "#D683CE" + }, + { + "alias": "/.*Echos -.*/", + "color": "#F9934E" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InEchoReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InEchoReps - Echo Reply messages received", + "refId": "G", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InEchos{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InEchos - Echo (request) messages received", + "refId": "H", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutEchoReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutEchoReps - Echo Reply messages sent", + "refId": "T", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutEchos{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutEchos - Echo (request) messages sent", + "refId": "U", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP Echos", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "fill": 2, + "height": "", + "id": 113, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "minSpan": 1, + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*Out.*/", + "transform": "negative-Y" + }, + { + "alias": "/.*AddrMaskReps.*/", + "color": "#B7DBAB" + }, + { + "alias": "/.*Masks -.*/", + "color": "#E5AC0E" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_netstat_Icmp_InAddrMaskReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InAddrMaskReps - Address Mask Reply messages received", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_InAddrMasks{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "InAddrMasks - Address Mask Request messages received", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutAddrMaskReps{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutAddrMaskReps - Address Mask Reply messages sent", + "refId": "Q", + "step": 4 + }, + { + "expr": "irate(node_netstat_Icmp_OutAddrMasks{instance=~\"$node:$port\",job=~\"$job\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "OutAddrMasks - Address Mask Request messages sent", + "refId": "R", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "ICMP Masks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Messages out (-) / in (+)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Network Netstat ICMP", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "375", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 40, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_duration_seconds{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape duration", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node Exporter Scrape Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "description": "", + "fill": 2, + "id": 157, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_scrape_collector_success{instance=~\"$node:$port\",job=~\"$job\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{collector}} - Scrape success", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Node Exporter Scrape Success", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Counter", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Node Exporter", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "vm-node-exporter", + "value": "vm-node-exporter" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": "label_values(node_exporter_build_info, job)", + "refresh": 1, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "11.3.1.5", + "value": "11.3.1.5" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": "label_values(node_exporter_build_info{job=~\"$job\"}, instance)", + "refresh": 1, + "regex": "/([^:]+):.*/", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "9100", + "value": "9100" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": "Port", + "multi": false, + "name": "port", + "options": [], + "query": "label_values(node_exporter_build_info{instance=~\"$node:(.*)\"}, instance)", + "refresh": 1, + "regex": "/[^:]+:(.*)/", + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Node Exporter Full", + "uid": "000000003" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/nodes.json b/deploy/grafana/dashboards/General/nodes.json new file mode 100644 index 000000000..5f05ed064 --- /dev/null +++ b/deploy/grafana/dashboards/General/nodes.json @@ -0,0 +1,860 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n (1 - rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"}[$__interval]))\n/ ignoring(cpu) group_left\n count without (cpu)( node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "interval": "1m", + "intervalFactor": 5, + "legendFormat": "{{cpu}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": {}, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "1m load average", + "refId": "A" + }, + { + "expr": "node_load5{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "5m load average", + "refId": "B" + }, + { + "expr": "node_load15{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "15m load average", + "refId": "C" + }, + { + "expr": "count(node_cpu_seconds_total{job=\"node-exporter\", instance=\"$instance\", mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "logical cores", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Load Average", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}\n-\n node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "node_memory_Buffers_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "refId": "B" + }, + { + "expr": "node_memory_Cached_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory cached", + "refId": "C" + }, + { + "expr": "node_memory_MemFree_bytes{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "memory free", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": {}, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 -\n(\n node_memory_MemAvailable_bytes{job=\"node-exporter\", instance=\"$instance\"}\n/\n node_memory_MemTotal_bytes{job=\"node-exporter\", instance=\"$instance\"}\n* 100\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Memory Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": {}, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/ read| written/", + "yaxis": 1 + }, + { + "alias": "/ io time/", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_disk_read_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} read", + "refId": "A" + }, + { + "expr": "rate(node_disk_written_bytes_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} written", + "refId": "B" + }, + { + "expr": "rate(node_disk_io_time_seconds_total{job=\"node-exporter\", instance=\"$instance\", device=~\"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}} io time", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "used", + "color": "#E0B400" + }, + { + "alias": "available", + "color": "#73BF69" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n max by (device) (\n node_filesystem_size_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n -\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "refId": "A" + }, + { + "expr": "sum(\n max by (device) (\n node_filesystem_avail_bytes{job=\"node-exporter\", instance=\"$instance\", fstype!=\"\"}\n )\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "available", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": {}, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Received", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "gridPos": {}, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{job=\"node-exporter\", instance=\"$instance\", device!=\"lo\"}[$__interval])", + "format": "time_series", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network Transmitted", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(node_exporter_build_info{job=\"node-exporter\"}, instance)", + "refresh": 2, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Nodes", + "uid": "fa49a4706d07a042595b664c87fb33ea" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/postgres-overview.json b/deploy/grafana/dashboards/General/postgres-overview.json new file mode 100644 index 000000000..3a5fdeec0 --- /dev/null +++ b/deploy/grafana/dashboards/General/postgres-overview.json @@ -0,0 +1,1258 @@ +{ + "annotations": { + "list": [] + }, + "description": "Performance metrics for Postgres", + "editable": true, + "gnetId": 455, + "graphTooltip": 0, + "hideControls": true, + "links": [], + "refresh": false, + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_fetched{datname=~\"$db\",instance=\"$host\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "fetched", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_returned{datname=~\"$db\",instance=\"$host\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "returned", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_inserted{datname=~\"$db\",instance=\"$host\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "inserted", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_updated{datname=~\"$db\",instance=\"$host\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "updated", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "fetched", + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_tup_deleted{datname=~\"$db\",instance=\"$host\"}[5m]))", + "format": "time_series", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deleted", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "tup_fetched" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 120, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rows", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 0, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "height": "55px", + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 2, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "dsType": "prometheus", + "expr": "sum(irate(pg_stat_database_xact_commit{datname=~\"$db\",instance=\"$host\"}[5m])) + sum(irate(pg_stat_database_xact_rollback{datname=~\"$db\",instance=\"$host\"}[5m]))", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "xact_commit" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [ + "10s" + ], + "type": "non_negative_derivative" + } + ] + ], + "step": 1800, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": "", + "title": "QPS", + "transparent": true, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": 1, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideZero": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_alloc{instance='$host'}[5m])", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_alloc", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend_fsync{instance='$host'}[5m])", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend_fsync", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_backend{instance='$host'}[5m])", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_backend", + "measurement": "postgresql", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_clean{instance='$host'}[5m])", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_clean", + "measurement": "postgresql", + "policy": "default", + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "Buffers Allocated", + "dsType": "prometheus", + "expr": "irate(pg_stat_bgwriter_buffers_checkpoint{instance='$host'}[5m])", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "buffers_checkpoint", + "measurement": "postgresql", + "policy": "default", + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "buffers_alloc" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Buffers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "conflicts", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_deadlocks{datname=~\"$db\",instance=\"$host\"}[5m]))", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "deadlocks", + "measurement": "postgresql", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "conflicts" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + }, + { + "alias": "deadlocks", + "dsType": "prometheus", + "expr": "sum(rate(pg_stat_database_conflicts{datname=~\"$db\",instance=\"$host\"}[5m]))", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "intervalFactor": 2, + "legendFormat": "conflicts", + "measurement": "postgresql", + "policy": "default", + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "deadlocks" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + }, + { + "params": [], + "type": "difference" + } + ] + ], + "step": 240, + "tags": [ + { + "key": "host", + "operator": "=~", + "value": "/^$host$/" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Conflicts/Deadlocks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": true, + "pointradius": 1, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=\"$host\"}) / (sum(pg_stat_database_blks_hit{datname=~\"$db\",instance=\"$host\"}) + sum(pg_stat_database_blks_read{datname=~\"$db\",instance=\"$host\"}))", + "intervalFactor": 2, + "legendFormat": "cache hit rate", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Cache hit ratio", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pg_stat_database_numbackends{datname=~\"$db\",instance=\"$host\"}", + "intervalFactor": 2, + "legendFormat": "{{__name__}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of active connections", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "postgres" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "monitor_postgres_exporter:9187", + "value": "monitor_postgres_exporter:9187" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "host", + "options": [], + "query": "label_values(up{job=~\".*postgres-exporter\"},instance)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "keycloak", + "value": "keycloak" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": true, + "label": "db", + "multi": false, + "name": "db", + "options": [], + "query": "label_values(pg_stat_database_tup_fetched{job=~\".*postgres-exporter\",instance=\"$host\",datname!~\"template.*|postgres\"},datname)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Postgres Overview", + "uid": "b_XuCbiWz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/prometheus-stats.json b/deploy/grafana/dashboards/General/prometheus-stats.json new file mode 100644 index 000000000..73082e4f8 --- /dev/null +++ b/deploy/grafana/dashboards/General/prometheus-stats.json @@ -0,0 +1,1352 @@ +{ + "annotations": { + "list": [] + }, + "description": "VIew Prometheus internal metrics: an extension of the standard dashboard. Useful for tuning, probably overkill for daily use.", + "editable": true, + "gnetId": 445, + "graphTooltip": 1, + "hideControls": false, + "links": [ + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Grafana Docs", + "tooltip": "", + "type": "link", + "url": "http://www.grafana.org/docs" + }, + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Prometheus Docs", + "type": "link", + "url": "http://prometheus.io/docs/introduction/overview/" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "100px", + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "decimals": 1, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(time() - process_start_time_seconds{job=\"prometheus\"})", + "intervalFactor": 2, + "refId": "A", + "step": 3600 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_local_storage_memory_series", + "intervalFactor": 2, + "refId": "A", + "step": 3600 + } + ], + "thresholds": "1,5", + "title": "Local Storage Memory Series", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_local_storage_indexing_queue_length", + "intervalFactor": 2, + "refId": "A", + "step": 3600 + } + ], + "thresholds": "500,4000", + "title": "Interal Storage Queue Length", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "Empty", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "prom-fed", + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_local_storage_checkpoint_last_duration_seconds", + "format": "time_series", + "intervalFactor": 2, + "metric": "prometheus_local_storage_checkpoint_duration_seconds", + "refId": "A", + "step": 3600 + } + ], + "thresholds": "", + "title": "Checkpoint duration", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "prometheus": "#F9BA8F", + "{instance=\"localhost:9090\",interval=\"5s\",job=\"prometheus\"}": "#F9BA8F" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_count[5m])", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Target Scrapes (last 5m)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}} ({{interval}})", + "metric": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scrape Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Scrapes", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_evaluator_duration_seconds{quantile!=\"0.01\", quantile!=\"0.05\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{quantile}}", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rule Eval Duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Rules", + "titleSize": "h6" + }, + { + "collapse": false, + "height": 227, + "panels": [ + { + "aliasColors": { + "prometheus": "#C15C17", + "{instance=\"localhost:9090\",job=\"prometheus\"}": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_local_storage_ingested_samples_total[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Samples ingested (rate:5m)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_local_storage_persistence_urgency_score", + "intervalFactor": 2, + "metric": "prometheus_local_storage_persistence_urgency_score", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Persistence Urgency", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_local_storage_rushed_mode", + "intervalFactor": 2, + "metric": "prometheus_local_storage_rushed_mode", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rushed mode", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_local_storage_chunks_to_persist", + "intervalFactor": 2, + "metric": "prometheus_local_storage_chunks_to_persist", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Chunks to persist", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 12, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_local_storage_chunk_ops_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Chunk ops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 13, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_local_storage_series_ops_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "metric": "prometheus_local_storage_series_ops_total", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Series ops", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "process_cpu_seconds_total", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_resident_memory_bytes", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}}", + "metric": "process_resident_memory_bytes", + "refId": "A", + "step": 600 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Resident memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "New row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-2d", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Prometheus Stats", + "uid": "pxmrjbiWk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/prometheus.json b/deploy/grafana/dashboards/General/prometheus.json new file mode 100644 index 000000000..4c045e046 --- /dev/null +++ b/deploy/grafana/dashboards/General/prometheus.json @@ -0,0 +1,1058 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "Count", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Uptime", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "instance", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Job", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "job", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Version", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "version", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "count by (job, instance, version) (prometheus_build_info{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + }, + { + "expr": "max by (job, instance) (time() - process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"})", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Prometheus Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Prometheus Stats", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m])) by (scrape_job) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Target Sync", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_sd_discovered_targets{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Targets", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Targets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Discovery", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_target_interval_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[5m]) / rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m]) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interval}} configured", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Scrape Interval Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_exceeded_sample_limit_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_bounds_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds: {{job}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "sum by (job) (rate(prometheus_target_scrapes_sample_out_of_order_total[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order: {{job}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Scrape failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Appended Samples", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Retrieval", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}} head series", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Series", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}} head chunks", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Storage", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(prometheus_engine_query_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",slice=\"inner_eval\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{job}} {{instance}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\",instance=~\"$instance\"}) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Stage Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "job", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(prometheus_build_info, job)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "instance", + "multi": true, + "name": "instance", + "options": [], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Prometheus", + "uid": "UQ0dk8PWk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/redis-dashboard-for-prometheus-redis-exporter.json b/deploy/grafana/dashboards/General/redis-dashboard-for-prometheus-redis-exporter.json new file mode 100644 index 000000000..5cea0374c --- /dev/null +++ b/deploy/grafana/dashboards/General/redis-dashboard-for-prometheus-redis-exporter.json @@ -0,0 +1,788 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Redis Dashboard for Prometheus Redis Exporter", + "editable": true, + "gnetId": 763, + "graphTooltip": 0, + "iteration": 1576747055178, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_Stateful", + "decimals": 0, + "editable": true, + "error": false, + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(max_over_time(redis_uptime_in_seconds{instance=~\"$instance\"}[$__interval]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "70%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus_Stateful", + "decimals": 0, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 0 + }, + "hideTimeOverride": true, + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "redis_connected_clients{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 2 + } + ], + "thresholds": "", + "timeFrom": "1m", + "timeShift": null, + "title": "Connected Clients", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": true, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(redis_keyspace_hits_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hits", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "irate(redis_keyspace_misses_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "misses", + "metric": "", + "refId": "B", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Hits / Misses per Sec", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max": "#BF1B00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "redis_memory_used_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "used", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "redis_memory_max_bytes{instance=~\"$instance\"} ", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "max", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 10, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(redis_net_input_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ input }}", + "refId": "A", + "step": 240 + }, + { + "expr": "rate(redis_net_output_bytes_total{instance=~\"$instance\"}[5m])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ output }}", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "fill": 7, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 5, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum (redis_db_keys{instance=~\"$instance\"}) by (db)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ db }} ", + "refId": "A", + "step": 240, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Items per DB", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "evicts": "#890F02", + "memcached_items_evicted_total{instance=\"172.17.0.1:9150\",job=\"prometheus\"}": "#890F02", + "reclaims": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 8, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "reclaims", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(redis_expired_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "expired", + "metric": "", + "refId": "A", + "step": 240, + "target": "" + }, + { + "expr": "sum(rate(redis_evicted_keys_total{instance=~\"$instance\"}[5m])) by (instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "evicted", + "refId": "B", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Expired / Evicted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "prometheus", + "redis" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "monitor_stateful_redis-exporter:9121", + "value": "monitor_stateful_redis-exporter:9121" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "instance", + "options": [ + { + "selected": true, + "text": "monitor_stateful_redis-exporter:9121", + "value": "monitor_stateful_redis-exporter:9121" + } + ], + "query": "monitor_stateful_redis-exporter:9121", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Redis Dashboard for Prometheus Redis Exporter", + "uid": "8QBoYnaWz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/secor-consumer-lag-and-kafka-topics-details.json b/deploy/grafana/dashboards/General/secor-consumer-lag-and-kafka-topics-details.json new file mode 100644 index 000000000..70bd3977d --- /dev/null +++ b/deploy/grafana/dashboards/General/secor-consumer-lag-and-kafka-topics-details.json @@ -0,0 +1,1161 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1577449788952, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_topic_partitions{topic=~\"$env.*\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Topics partition count", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_topic_partition_current_offset{topic=~\"$env.*\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Topics current offset", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 10 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.ingestion.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.ingestion.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 11, + "y": 10 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.channel.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.channel.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 20 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.raw.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.raw.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 11, + "y": 20 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.failed.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.failed.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 9, + "w": 11, + "x": 0, + "y": 29 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.learning.failed.events.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.learning.failed.events.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 11, + "y": 29 + }, + "id": 18, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.denorm.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.denorm.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 38 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.unique.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.unique.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 11, + "y": 38 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.graph.events.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.graph.events.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 11, + "x": 0, + "y": 48 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.pipeline.metrics\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.pipeline_metrics", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_Stateful", + "fill": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 11, + "y": 48 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kafka_consumergroupzookeeper_lag_zookeeper{consumergroup=~\"$env.telemetry.derived.backup\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}}::{{topic}}::{{partition}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$env.telemetry.derived.backup", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "ntpstaging", + "value": "ntpstaging" + }, + "hide": 2, + "label": null, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "skipUrlSync": false, + "type": "constant" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Secor Consumer Lag and kafka topics details", + "uid": "C1Bnx7-Zk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/server.json b/deploy/grafana/dashboards/General/server.json new file mode 100644 index 000000000..5e7dfb5a8 --- /dev/null +++ b/deploy/grafana/dashboards/General/server.json @@ -0,0 +1,208 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "prom-fed", + "fill": 1, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes_total{device='eth0',instance!='11.3.3.23:9100',cluster='$cluster'}[2m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Egress: {{cluster}}: {{instance}}", + "refId": "A", + "step": 30 + }, + { + "expr": "rate(node_network_receive_bytes_total{device='eth0',instance!='11.3.3.23:9100',cluster='$cluster'}[2m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Ingress: {{cluster}}: {{instance}}", + "refId": "B", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "DC3", + "value": "DC3" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": "swarm", + "multi": false, + "name": "cluster", + "options": [ + { + "selected": false, + "text": "DC2", + "value": "DC2" + }, + { + "selected": true, + "text": "DC3", + "value": "DC3" + } + ], + "query": "node_load1", + "refresh": 0, + "regex": "/(DC.)/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "11.3.1.6", + "value": "11.3.1.6" + }, + "datasource": "prom-fed", + "hide": 0, + "includeAll": false, + "label": "server", + "multi": false, + "name": "server_ip", + "options": [], + "query": "node_load1{cluster=~'DC.'}", + "refresh": 1, + "regex": "/=\"(\\d+.*\\d):/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Server", + "uid": "000000002" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/service-memory-usage-ds1.json b/deploy/grafana/dashboards/General/service-memory-usage-ds1.json new file mode 100644 index 000000000..62618d9a4 --- /dev/null +++ b/deploy/grafana/dashboards/General/service-memory-usage-ds1.json @@ -0,0 +1,1249 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"player_player\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "player_player", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"content-service\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "content-service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 3, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"learner-service\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "learner-service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 4, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"telemetry-service\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "telemetry-service", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 5, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"telemetry-logstash\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "telemetry-logstash", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"Prometheus_DS1\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Prometheus_DS1", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 8, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"logger_kibana\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "logger_kibana", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 7, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"proxy_proxy\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "proxy_proxy", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 9, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"api-manager_kong\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "api-manager_kong", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 10, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"api-manager_echo\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "api-manager_echo", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 11, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"logger_logspout\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "logger_logspout", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 12, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"monitor_logstash\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "monitor_logstash", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 13, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"logger_logstash\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "logger_logstash", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 14, + "legend": { + "avg": false, + "current": true, + "max": true, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(container_memory_usage_bytes{container_label_com_docker_swarm_service_name=~\"monitor_cadvisor\"})", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "monitor_cadvisor", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Service memory usage DS1", + "uid": "_lAAXtZZz" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/use-method-cluster.json b/deploy/grafana/dashboards/General/use-method-cluster.json new file mode 100644 index 000000000..8b7d7fafd --- /dev/null +++ b/deploy/grafana/dashboards/General/use-method-cluster.json @@ -0,0 +1,846 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_cpu_utilisation:rate1m{job=\"node-exporter\"}\n*\n instance:node_num_cpu:sum{job=\"node-exporter\"}\n/ ignoring (instance) group_left\n sum without (instance) (instance:node_num_cpu:sum{job=\"node-exporter\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_load1_per_cpu:ratio{job=\"node-exporter\"}\n/ ignoring (instance) group_left\n count without (instance) (instance:node_load1_per_cpu:ratio{job=\"node-exporter\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (load1 per CPU)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance:node_memory_utilisation:ratio{job=\"node-exporter\"}\n/ ignoring (instance) group_left\n count without (instance) (instance:node_memory_utilisation:ratio{job=\"node-exporter\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Receive", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Transmit", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/ Receive/", + "stack": "A" + }, + { + "alias": "/ Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Receive", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} Transmit", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Network", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"}\n/ ignoring (instance, device) group_left\n count without (instance, device) (instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{device}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"}\n/ ignoring (instance, device) group_left\n count without (instance, device) (instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}} {{device}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without (device) (\n max without (fstype, mountpoint) (\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"} - node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\"}\n )\n ) \n/ ignoring (instance) group_left\n sum without (instance, device) (\n max without (fstype, mountpoint) (\n node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\"}\n )\n )\n) \n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "legendLink": "/dashboard/file/node-rsrc-use.json", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "USE Method / Cluster", + "uid": "3e97d1d02672cdd0861f4c97c64f89b2" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/use-method-node.json b/deploy/grafana/dashboards/General/use-method-node.json new file mode 100644 index 000000000..88a030f8d --- /dev/null +++ b/deploy/grafana/dashboards/General/use-method-node.json @@ -0,0 +1,869 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_cpu_utilisation:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Utilisation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_load1_per_cpu:ratio{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Saturation", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Saturation (Load1 per CPU)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "CPU", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_memory_utilisation:ratio{job=\"node-exporter\", job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_vmstat_pgmajfault:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major page faults", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory Saturation (Major Page Faults)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Memory", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive", + "legendLink": null, + "step": 10 + }, + { + "expr": "instance:node_network_transmit_bytes_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Net Utilisation (Bytes Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Receive/", + "stack": "A" + }, + { + "alias": "/Transmit/", + "stack": "B", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance:node_network_receive_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Receive drops", + "legendLink": null, + "step": 10 + }, + { + "expr": "instance:node_network_transmit_drop_excluding_lo:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Transmit drops", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Net Saturation (Drops Receive/Transmit)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "rps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Net", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "instance_device:node_disk_io_time_weighted_seconds:rate1m{job=\"node-exporter\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk IO Saturation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk IO", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1 -\n(\n max without (mountpoint, fstype) (node_filesystem_avail_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n/\n max without (mountpoint, fstype) (node_filesystem_size_bytes{job=\"node-exporter\", fstype!=\"\", instance=\"$instance\"})\n)\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{device}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk Space Utilisation", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Disk Space", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(up{job=\"node-exporter\"}, instance)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "USE Method / Node", + "uid": "fac67cfbe174d3ef53eb473d73d9212f" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/General/weekly-report-dashboard-ds1.json b/deploy/grafana/dashboards/General/weekly-report-dashboard-ds1.json new file mode 100644 index 000000000..cbd22b8dd --- /dev/null +++ b/deploy/grafana/dashboards/General/weekly-report-dashboard-ds1.json @@ -0,0 +1,1844 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1558376084165, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "panels": [], + "repeat": null, + "title": "Daily Peak of Request per Second", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_request_status_count[5m]))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "All", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Daily Peaks of Request per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 17, + "panels": [], + "repeat": null, + "title": "Max % HTTP 4xx Errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|5..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max % HTTP 4xx Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 4, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(nginx_request_status_count{status_code!~\"2..|3..|4..\"}[5m])) by (status_code) / ignoring(status_code) group_left sum(increase(nginx_request_status_count[5m]))) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max % HTTP 5xx Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 18, + "panels": [], + "repeat": null, + "title": "API Requests Error %", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{status_code!~\"2..|3..|5..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "4xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(increase(kong_request_status_count{status_code!~\"2..|3..|4..\"}[5m])) by (api, status_code) / ignoring(api, status_code) group_left sum(increase(kong_request_status_count[5m])))", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}-{{status_code}}", + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "5xx Error %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 19, + "panels": [], + "repeat": null, + "title": "Load Avg on VMs", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(host) * on (instance) group_left(nodename) node_load1", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{host}}", + "refId": "A", + "step": 120 + }, + { + "expr": "(node_load1) * on (instance) group_left(nodename) node_uname_info{job=\"vm-node-exporter\"}", + "format": "time_series", + "hide": false, + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{nodename}}", + "refId": "B", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load avg on VMs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 20, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_user_seconds_total{image!=\"\", container_label_com_docker_swarm_service_name=~\"$service\"}[5m])) by (name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "cpu", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Load Avg of Containers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 21, + "panels": [], + "repeat": null, + "title": "Top 10 API Requests", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(kong_request_count[5m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "metric": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Top 10 API Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 22, + "panels": [], + "repeat": null, + "title": "APIs Avg/Min/Max Latency", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus_DS1", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kong_latency_time", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{api}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "APIs Average/Min/Max Response Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 23, + "panels": [], + "repeat": null, + "title": "Publish Pipeline Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 10, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "publish-pipeline" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Publish Pipeline Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 11, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "publish-pipeline" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Publish Pipeline Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 24, + "panels": [], + "repeat": null, + "title": "Composite Indexer Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 12, + "interval": "> 300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "hide": false, + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "composite-search-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Composite Indexer Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#C15C17" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 65 + }, + "id": 13, + "interval": "> 300s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "policy": "default", + "query": "SELECT \"error\", \"failed\" FROM \"pipeline_metrics\" WHERE \"env\" =~ /^$env$/ AND \"job_name\" = 'composite-search-indexer' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "composite-search-indexer" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Composite Indexer Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 25, + "panels": [], + "repeat": null, + "title": "Telemetry Metrics", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 14, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"success\", \"skipped\" FROM \"pipeline_metrics\" WHERE \"env\" = 'ntpprod' AND \"job_name\" = 'TelemetryValidator' AND $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "success" + ], + "type": "field" + } + ], + [ + { + "params": [ + "skipped" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=", + "value": "production" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryValidator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Validator Success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "pipeline_metrics.error": "#890F02", + "pipeline_metrics.failed": "#E0752D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB_Datapipeline", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 73 + }, + "id": 15, + "interval": ">300s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [], + "measurement": "pipeline_metrics", + "orderByTime": "ASC", + "policy": "autogen", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "error" + ], + "type": "field" + } + ], + [ + { + "params": [ + "failed" + ], + "type": "field" + } + ] + ], + "tags": [ + { + "key": "env", + "operator": "=~", + "value": "/^$env$/" + }, + { + "condition": "AND", + "key": "job_name", + "operator": "=", + "value": "TelemetryValidator" + } + ] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Telemetry Validator Failure", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "ntpprod", + "value": "ntpprod" + }, + "hide": 0, + "includeAll": false, + "label": "Environment", + "multi": false, + "name": "env", + "options": [ + { + "selected": true, + "text": "ntpstaging", + "value": "ntpstaging" + } + ], + "query": "ntpstaging", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus_DS1", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Service", + "multi": true, + "name": "service", + "options": [], + "query": "label_values(container_last_seen, container_label_com_docker_swarm_service_name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now/d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Weekly Report Dashboard DS1", + "uid": "9C10XtWZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/istio/ingress-dashboard.json b/deploy/grafana/dashboards/istio/ingress-dashboard.json new file mode 100644 index 000000000..3f3aae868 --- /dev/null +++ b/deploy/grafana/dashboards/istio/ingress-dashboard.json @@ -0,0 +1,895 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1578368835070, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 70, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Requests by Destination And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 72, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Request Duration by Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 71, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Success Rate (non-5xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 75, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"[2|3].*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"[2|3].*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Error Rate (non-2xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 73, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Request Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 74, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Destination Service", + "multi": true, + "name": "dstsvc", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service))", + "refresh": 1, + "regex": "/.*destination_service=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 4, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ingress Dashboard", + "uid": "_q3klkPZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/istio/istio-ingress-dashboard.json b/deploy/grafana/dashboards/istio/istio-ingress-dashboard.json new file mode 100644 index 000000000..f2778042e --- /dev/null +++ b/deploy/grafana/dashboards/istio/istio-ingress-dashboard.json @@ -0,0 +1,942 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1578381966806, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 70, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Traffic: Request / Sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 13, + "x": 0, + "y": 6 + }, + "id": 71, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Success Rate (non-5xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 11, + "x": 13, + "y": 6 + }, + "id": 75, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\",response_code!~\"(2|3).*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}:{{response_code}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"istio-system\", source_workload=~\"istio-ingressgateway\", reporter=\"source\",response_code!~\"(2|3).*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}:{{response_code}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Error Rate (non-2|3xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 12 + }, + "id": 72, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Duration by Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 12 + }, + "id": 73, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 74, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"istio-ingressgateway\", source_workload_namespace=~\"istio-system\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "text": "staging", + "value": "staging" + }, + "datasource": "Prometheus", + "definition": "query_result(sum(istio_requests_total) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total) by (destination_workload_namespace))", + "hide": 2, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(sum(istio_requests_total) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total) by (destination_workload_namespace))", + "refresh": 1, + "regex": "/.*_namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "adminutil", + "value": "adminutil" + }, + "datasource": "Prometheus", + "definition": "query_result((sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_requests_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)) or (sum(istio_tcp_sent_bytes_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)))", + "hide": 2, + "includeAll": false, + "label": "Workload", + "multi": false, + "name": "workload", + "options": [], + "query": "query_result((sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_requests_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)) or (sum(istio_tcp_sent_bytes_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Destination Service", + "multi": true, + "name": "dstsvc", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service))", + "refresh": 1, + "regex": "/.*destination_service=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 4, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Ingress Dashboard", + "uid": "0r4BqkEZk" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/istio/istio-mesh-grafana.json b/deploy/grafana/dashboards/istio/istio-mesh-grafana.json new file mode 100644 index 000000000..e8ace0f24 --- /dev/null +++ b/deploy/grafana/dashboards/istio/istio-mesh-grafana.json @@ -0,0 +1,874 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "grafana" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=~\"destination\"}[1m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Global Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Global Success Rate (non-5xx responses)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"4.*\"}[1m])) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "4xxs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 23, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"5.*\"}[1m])) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "5xxs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 21, + "w": 24, + "x": 0, + "y": 3 + }, + "hideTimeOverride": false, + "id": 73, + "links": [], + "pageSize": null, + "repeat": null, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 4, + "desc": true + }, + "styles": [ + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Workload grafana", + "linkUrl": "/grafana/db/istio-workload-grafana?var-namespace=$__cell_2&var-workload=$__cell_", + "pattern": "destination_workload", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "Success Rate", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + ".95", + " 1.00" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell grafana", + "linkUrl": "/grafana/d/istio-workload-grafana?var-workload=$__cell_2&var-namespace=$__cell_3", + "pattern": "destination_workload_var", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Service", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell grafana", + "linkUrl": "/grafana/d/istio-service-grafana?var-service=$__cell", + "pattern": "destination_service", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "destination_workload_namespace", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "label_join(sum(rate(istio_requests_total{reporter=\"destination\", response_code=\"200\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", + "refId": "A" + }, + { + "expr": "label_join((histogram_quantile(0.50, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.50, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", + "refId": "B" + }, + { + "expr": "label_join((histogram_quantile(0.90, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.90, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "D" + }, + { + "expr": "label_join((histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.99, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "E" + }, + { + "expr": "label_join((sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) by (destination_workload, destination_workload_namespace) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m])) by (destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "F" + } + ], + "timeFrom": null, + "title": "HTTP/GRPC Workloads", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 24 + }, + "hideTimeOverride": false, + "id": 109, + "links": [], + "pageSize": null, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "$__cell grafana", + "linkUrl": "/grafana/db/istio-tcp-workload-grafana?var-namespace=$__cell_2&&var-workload=$__cell", + "pattern": "destination_workload", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bytes Sent", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bytes Received", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell grafana", + "linkUrl": "/grafana/db/istio-workload-grafana?var-namespace=$__cell_3&var-workload=$__cell_2", + "pattern": "destination_workload_var", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "destination_workload_namespace", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Service", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell grafana", + "linkUrl": "/grafana/db/istio-service-grafana?var-service=$__cell", + "pattern": "destination_service", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "label_join(sum(rate(istio_tcp_received_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}", + "refId": "C" + }, + { + "expr": "label_join(sum(rate(istio_tcp_sent_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "TCP Workloads", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 111, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build) by (component, tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ component }}: {{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Istio Components by Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Istio Mesh grafana", + "uid": "istio" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/istio/istio-service-dashboard.json b/deploy/grafana/dashboards/istio/istio-service-dashboard.json new file mode 100644 index 000000000..c2e013e0b --- /dev/null +++ b/deploy/grafana/dashboards/istio/istio-service-dashboard.json @@ -0,0 +1,2620 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1577942154932, + "links": [], + "panels": [ + { + "content": "
\nSERVICE: $service\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 89, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Client Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Client Success Rate (non-5xx responses)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 87, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Client Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Received Bytes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 97, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Server Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 98, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Server Success Rate (non-5xx responses)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 99, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Server Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 100, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m])) ", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Sent Bytes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "content": "
\nCLIENT WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 45, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"source\",source_workload=~\"$srcwl\",source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"source\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Requests by Source And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 80, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "
\nSERVICE WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 69, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 90, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"destination\",destination_workload=~\"$dstwl\",destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"destination\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Requests by Destination And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 91, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 94, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 95, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 96, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Service", + "multi": false, + "name": "service", + "options": [], + "query": "label_values(destination_service)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Client Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Client Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Service Workload Namespace", + "multi": true, + "name": "dstns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (destination_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Service Workload", + "multi": true, + "name": "dstwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Service Dashboard", + "uid": "istio-service-grafana" +} \ No newline at end of file diff --git a/deploy/grafana/dashboards/istio/istio-workload-dashboard.json b/deploy/grafana/dashboards/istio/istio-workload-dashboard.json new file mode 100644 index 000000000..dcc96af9d --- /dev/null +++ b/deploy/grafana/dashboards/istio/istio-workload-dashboard.json @@ -0,0 +1,2340 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1578352252798, + "links": [], + "panels": [ + { + "content": "
\nWORKLOAD: $workload.$namespace\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 89, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Incoming Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Incoming Success Rate (non-5xx responses)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 87, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Server Traffic", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 85, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Client Traffic", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "content": "
\nINBOUND WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 45, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Requests by Source And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 80, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "
\nOUTBOUND SERVICES\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 69, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 70, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Requests by Destination And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 71, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Success Rate (non-5xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 72, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Request Duration by Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 73, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Request Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 74, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Sent on Outgoing TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 78, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Received from Outgoing TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(sum(istio_requests_total) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total) by (destination_workload_namespace))", + "refresh": 1, + "regex": "/.*_namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Workload", + "multi": false, + "name": "workload", + "options": [], + "query": "query_result((sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_requests_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)) or (sum(istio_tcp_sent_bytes_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Destination Service", + "multi": true, + "name": "dstsvc", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service))", + "refresh": 1, + "regex": "/.*destination_service=\"([^\"]*).*/", + "skipUrlSync": false, + "sort": 4, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Workload Dashboard", + "uid": "UbsSZTDik" +} \ No newline at end of file diff --git a/deploy/jenkins/jenkins-jobs-setup.sh b/deploy/jenkins/jenkins-jobs-setup.sh index 8294a629e..4cfb90ec7 100755 --- a/deploy/jenkins/jenkins-jobs-setup.sh +++ b/deploy/jenkins/jenkins-jobs-setup.sh @@ -37,6 +37,16 @@ setupJobs(){ find $JENKINS_TMP/Deploy/jobs/${arr[$key]}/jobs/Summary/jobs/DeployedVersions -type f -name config.xml -exec sed -i "s#Deploy/${arr[0]}/#Deploy/${arr[$key]}/#g" {} \; echo -e "\e[0;33m${bold}Jobs created for ${arr[$key]}${normal}" done + echo -e "\e[0;36m${bold}Do you want to disable auto trigger of build jobs based on new commits?${normal}" + read -p 'y/n: ' choice + if [[ $choice == "y" ]]; then + find $JENKINS_TMP/Build -type f -name config.xml -exec sed -i 's/.*<\/spec>/<\/spec>/g' {} \; + fi + echo -e "\e[0;36m${bold}Do you want to disable daily backup jobs (Ex: DB backups)?${normal}" + read -p 'y/n: ' choice + if [[ $choice == "y" ]]; then + find $JENKINS_TMP/OpsAdministration -type f -name config.xml -exec sed -i 's/.*<\/spec>/<\/spec>/g' {} \; + fi diffs=$(colordiff -r --suppress-common-lines --no-dereference -x 'nextBuildNumber' -x 'builds' -x 'last*' /var/lib/jenkins/jobs $JENKINS_TMP | wc -l) if [[ $diffs -eq 0 ]]; then echo -e "\e[0;33m${bold}No changes detected. Exiting...${normal}" @@ -52,6 +62,7 @@ echo -e "\e[0;33m${bold}This might take a while... Do not kill the process!${nor if [[ $changes == "YES" ]]; then rsync -r $JENKINS_TMP/* /var/lib/jenkins/jobs chown -R jenkins:jenkins /var/lib/jenkins/jobs + echo -e "\e[0;32m${bold}Setup complete!${normal}" else echo -e "\e[0;31m${bold}Aborted!${normal}" fi diff --git a/deploy/jenkins/jenkins-plugins-setup.sh b/deploy/jenkins/jenkins-plugins-setup.sh index ee6d0744f..1bc665b7e 100755 --- a/deploy/jenkins/jenkins-plugins-setup.sh +++ b/deploy/jenkins/jenkins-plugins-setup.sh @@ -2,20 +2,25 @@ bold=$(tput bold) normal=$(tput sgr0) -echo -e "\e[0;32m${bold}Jenkins configuration setup...${normal}" -read -p 'Jenkins URL without http://: ' server -read -p 'Jenkins admin username: ' username -read -sp 'Jenkins admin Password: ' password - if [[ ! -d /var/lib/jenkins/.m2 ]]; then echo -e "\n\e[0;32m${bold}Downloading and copying m2 directory to Jenkins ${normal}" -wget https://sunbirdpublic.blob.core.windows.net/installation/m2_new.zip -unzip m2_new.zip +wget https://sunbirdpublic.blob.core.windows.net/installation/m2_updated.zip +unzip m2_updated.zip mv .m2 /var/lib/jenkins chown -R jenkins:jenkins /var/lib/jenkins/.m2 fi -echo -e "\n\e[0;32m${bold}Installating plugins... ${normal}" -./butler plugins i -s $server -u $username -p $password +echo -e "\n\e[0;32m${bold}Downloading and copying jenkins plugin directory to Jenkins ${normal}" +if [[ ! -d /var/lib/jenkins/plugins ]]; then +wget https://sunbirdpublic.blob.core.windows.net/installation/JenkinsPlugin.zip +unzip JenkinsPlugin.zip +mv plugins /var/lib/jenkins/ +chown -R jenkins:jenkins /var/lib/jenkins/plugins +else +wget https://sunbirdpublic.blob.core.windows.net/installation/JenkinsPlugin.zip +unzip JenkinsPlugin.zip +cp -rf plugins/* /var/lib/jenkins/plugins/ +chown -R jenkins:jenkins /var/lib/jenkins/plugins +fi echo -e "\n\e[0;32m${bold}Go to manage jenkins -> Plugin manager -> Update center -> Check status of installation${normal}" diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index 979438348..c28e87690 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -46,27 +46,27 @@ apt-get update apt-get install -y docker-ce docker-ce-cli containerd.io echo -e "\n\e[0;32m${bold}Installating node and npm modules" -wget https://nodejs.org/download/release/v6.1.0/node-v6.1.0-linux-x64.tar.gz -tar -xf node-v6.1.0-linux-x64.tar.gz -rm -rf /usr/local/lib/node-v6.1.0-linux-x64 +wget https://nodejs.org/download/release/v6.17.1/node-v6.17.1-linux-x64.tar.gz +tar -xf node-v6.17.1-linux-x64.tar.gz +rm -rf /usr/local/lib/node-v6.17.1-linux-x64 rm -rf /usr/bin/node rm -rf /usr/bin/npm rm -rf /usr/bin/grunt rm -rf /usr/bin/bower rm -rf /usr/bin/gulp -mv node-v6.1.0-linux-x64 /usr/local/lib/ -ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/node /usr/bin/node -ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/npm /usr/bin/npm +mv node-v6.17.1-linux-x64 /usr/local/lib/ +ln -s /usr/local/lib/node-v6.17.1-linux-x64/bin/node /usr/bin/node +ln -s /usr/local/lib/node-v6.17.1-linux-x64/bin/npm /usr/bin/npm npm install -g grunt-cli@1.2.0 -ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/grunt /usr/bin/grunt +ln -s /usr/local/lib/node-v6.17.1-linux-x64/bin/grunt /usr/bin/grunt npm install -g bower@1.8.0 -ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/bower /usr/bin/bower +ln -s /usr/local/lib/node-v6.17.1-linux-x64/bin/bower /usr/bin/bower npm install -g gulp@3.9.1 -ln -s /usr/local/lib/node-v6.1.0-linux-x64/bin/gulp /usr/bin/gulp -rm -rf node-v6.1.0-linux-x64* +ln -s /usr/local/lib/node-v6.17.1-linux-x64/bin/gulp /usr/bin/gulp +rm -rf node-v6.17.1-linux-x64* echo -e "\n\e[0;32m${bold}Installating Ansible${normal}" -pip install ansible==2.5.0 +pip install ansible==2.5.2 echo -e "\n\e[0;32m${bold}Installating azure cli${normal}" apt-get install ca-certificates curl apt-transport-https lsb-release gnupg @@ -110,6 +110,24 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34. echo -e "\n\e[0;32m${bold}Installing jmespath${normal}" pip install jmespath +#### Kubernetes Tools #### + +# Install Helm version 3.0.2 +echo -e "\n\e[0;32m${bold}Installating Helm${normal}" +wget https://get.helm.sh/helm-v3.0.2-linux-386.tar.gz +tar -xzvf helm-v3.0.2-linux-386.tar.gz +rm -rf /usr/local/bin/helm +cp linux-386/helm /usr/local/bin/helm +rm -rf helm-v* linux-amd* + +# Install kubectl +echo -e "\n\e[0;32m${bold}Installating kubectl${normal}" +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +touch /etc/apt/sources.list.d/kubernetes.list +echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list +apt-get update +apt-get install -y kubectl + echo -e "\n\e[0;32m${bold}Clean up${normal}" sudo apt -y autoremove diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/config.xml index 68477b8ac..ca27afff7 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Assessment/config.xml new file mode 100644 index 000000000..99461aa98 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Assessment/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/Assessment + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/Assessment + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cassandra/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cassandra/config.xml index 08e2a6496..e26714086 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cassandra/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cassandra/config.xml @@ -1,7 +1,7 @@ - + - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -20,7 +20,7 @@ - + false false @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 @@ -105,4 +105,4 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CassandraDecrypt/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CassandraDecrypt/config.xml index ad44ba395..1dec9c818 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CassandraDecrypt/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CassandraDecrypt/config.xml @@ -1,5 +1,5 @@ - + @@ -20,7 +20,7 @@ - + false false @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cert/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cert/config.xml new file mode 100644 index 000000000..fc0675046 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Cert/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/Cert + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/Cert + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml new file mode 100644 index 000000000..418c706b3 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/CertRegistryService + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/CertRegistryService + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Content/config.xml index ee29acd2a..ff83188c2 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Content/config.xml @@ -1,7 +1,7 @@ - + - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -20,7 +20,7 @@ - + false false @@ -75,8 +75,8 @@ ArtifactRepo - Push the docker image to container registry. - - + + 2 @@ -95,7 +95,7 @@ ArtifactRepo - Push the docker image to container registry. true false - 1 + 0 false @@ -105,4 +105,4 @@ ArtifactRepo - Push the docker image to container registry. false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/EncService/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/EncService/config.xml new file mode 100644 index 000000000..77c780c03 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/EncService/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/EncService + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/EncService + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml new file mode 100644 index 000000000..1d398f884 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/KnowledgeMW + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/KnowledgeMW + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 1 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Notification/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Notification/config.xml new file mode 100644 index 000000000..a6f84bf6e --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Notification/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/Notification + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/Notification + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Print/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Print/config.xml new file mode 100644 index 000000000..df8959b6a --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/Print/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/Print + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/Print + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 1 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/UserOrg/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/UserOrg/config.xml new file mode 100644 index 000000000..26c03bc49 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/Core/jobs/UserOrg/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/Core/UserOrg + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/Core/UserOrg + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/config.xml index 37e2c9b17..ca27afff7 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/config.xml @@ -3,7 +3,7 @@ - + diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Analytics/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Analytics/config.xml index b4671cf17..7010abb55 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Analytics/config.xml @@ -1,5 +1,5 @@ - + @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml new file mode 100644 index 000000000..2f3f7f3cd --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/DataPipeline/ApiModule + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + Build/DataPipeline/ApiModule + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 1 + false + + + + pipelines/upload/docker/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/DataPipeline/config.xml index b7fac1425..d3a16f329 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/DataPipeline/config.xml @@ -1,5 +1,5 @@ - + @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml new file mode 100644 index 000000000..309207380 --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml @@ -0,0 +1,108 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/DataPipeline/ETLJobs + false + + + build_number + <font color=darkgreen size=2><b>OPTIONAL: Specify the build job number to upload / copy the artifact built in that job.</b></font> + lastSuccessfulBuild + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - Just keep a copy of the artifact in Jenkins.</b></font> + + + ArtifactRepo + JenkinsJob + + + + + + + 0 + 0 + + false + project + false + + + + + + + /Build/DataPipeline/ETLJobs + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${public_repo_branch} + + + false + + + + true + false + + 1 + false + + + + pipelines/upload/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml index c7a78032b..7a140ec41 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml @@ -1,7 +1,7 @@ - + - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -20,7 +20,7 @@ - + false false @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 @@ -105,4 +105,4 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml index d417bc70e..5444de21f 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml @@ -1,5 +1,5 @@ - + @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/SyncTool/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/SyncTool/config.xml index b721b0316..02ab22706 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/SyncTool/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/SyncTool/config.xml @@ -1,5 +1,5 @@ - + @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml index da60236ff..2d6034541 100644 --- a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -75,8 +75,8 @@ ArtifactRepo - Upload and keep a copy of the artifact in Jenkins, JenkinsJob - J - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Core/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/config.xml index 37e2c9b17..ca27afff7 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/config.xml @@ -3,7 +3,7 @@ - + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml index da41923c8..2ee272e18 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/APIManager/config.xml @@ -1,5 +1,5 @@ - + false @@ -8,6 +8,16 @@ false false + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + 0 0 @@ -18,13 +28,13 @@ - + false - + false CassandraTrigger + Build/Core/CassandraTrigger ET_FORMATTED_HTML @@ -71,17 +72,18 @@ return """<b>This parameter is not used</b>"&quo - + H/15 * * * * false - + + - - + + 2 - https://github.com/project-sunbird/sunbird-utils.git + https://github.com/project-sunbird/sunbird-learning-platform.git @@ -106,4 +108,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Cert/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Cert/config.xml new file mode 100644 index 000000000..a867909a6 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Cert/config.xml @@ -0,0 +1,85 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + + + github_release_tag + <font color=blue size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/cert-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/CertRegistryService/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/CertRegistryService/config.xml new file mode 100644 index 000000000..bcd353ae7 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/CertRegistryService/config.xml @@ -0,0 +1,85 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + + + github_release_tag + <font color=blue size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/certificate-registry.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml index 715da0d80..34533b223 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml @@ -1,6 +1,13 @@ - - + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + false @@ -39,18 +46,18 @@ - + 15 * * * * false - - + + 2 - https://github.com/project-sunbird/sunbird-content-service.git + https://github.com/project-sunbird/knowledge-platform.git @@ -70,7 +77,7 @@ - Jenkinsfile + build/content-service/Jenkinsfile false diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/EncService/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/EncService/config.xml new file mode 100644 index 000000000..013e1a9f8 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/EncService/config.xml @@ -0,0 +1,84 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 10 + + + + false + false + + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/enc-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Keycloak/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Keycloak/config.xml index ebec82183..75e3f824e 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Keycloak/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Keycloak/config.xml @@ -1,5 +1,5 @@ - + false @@ -39,14 +39,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/KnowledgeMW/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/KnowledgeMW/config.xml new file mode 100644 index 000000000..71e5de7bb --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/KnowledgeMW/config.xml @@ -0,0 +1,78 @@ + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-content-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml index df134e70a..b8d0132a3 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml @@ -1,5 +1,5 @@ - + @@ -46,14 +46,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml index 69ca28496..417df3f39 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml @@ -1,5 +1,5 @@ - + @@ -47,12 +47,12 @@ - - + + 2 - https://github.com/project-sunbird/sunbird-course-service.git + https://github.com/project-sunbird/sunbird-course-service diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Notification/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Notification/config.xml new file mode 100644 index 000000000..511ab2d2e --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Notification/config.xml @@ -0,0 +1,85 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + + + github_release_tag + <font color=blue size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-notification-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Player/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Player/config.xml index 903757eea..4558b6a43 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Player/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Player/config.xml @@ -1,5 +1,5 @@ - + @@ -35,19 +35,19 @@ sunbird_content_editor_artifact_url <font color=dimgray size=2><b>Blob URL for content editor zip file</b></font> - https://sunbirddev.blob.core.windows.net/sunbird-content-dev/artefacts/editor/content-editor-iframe-2.5.0.zip + false sunbird_collection_editor_artifact_url <font color=dimgray size=2><b>Blob URL for collection editor zip file</b></font> - https://sunbirddev.blob.core.windows.net/sunbird-content-dev/artefacts/editor/collection-editor-iframe-2.5.0.zip + false sunbird_generic_editor_artifact_url <font color=dimgray size=2><b>Blob URL for generic editor zip file</b></font> - https://sunbirddev.blob.core.windows.net/sunbird-content-dev/artefacts/editor/generic-editor-iframe-2.5.0.zip + false @@ -64,14 +64,14 @@ - + H/15 * * * * false - - + + 2 @@ -100,4 +100,4 @@ false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Print/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Print/config.xml new file mode 100644 index 000000000..2ec508159 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Print/config.xml @@ -0,0 +1,78 @@ + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/print-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Telemetry/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Telemetry/config.xml index 833132b7d..8db1eb374 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Telemetry/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Telemetry/config.xml @@ -1,5 +1,5 @@ - + @@ -20,7 +20,7 @@ - + false false @@ -42,18 +42,19 @@ project false + 1 - + H/15 * * * * false - - + + 2 @@ -81,5 +82,5 @@ false - false + true \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/UserOrg/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/UserOrg/config.xml new file mode 100644 index 000000000..3b5cc656b --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/UserOrg/config.xml @@ -0,0 +1,84 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 10 + + + + false + false + + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-user-service.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Yarn/config.xml index 1edcb16ff..76a1d0c7f 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -42,18 +42,19 @@ project false + 1 - + H/15 * * * * false - - + + 2 @@ -81,5 +82,5 @@ false - false + true \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/config.xml index c3548f16c..2b6dfdd25 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/config.xml @@ -3,7 +3,7 @@ - + diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Analytics/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Analytics/config.xml index ea682d2b7..ce25972cb 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Analytics/config.xml @@ -1,5 +1,5 @@ - + @@ -46,14 +46,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ApiModule/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ApiModule/config.xml new file mode 100644 index 000000000..79fbc9b23 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ApiModule/config.xml @@ -0,0 +1,85 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 1 + + + + + false + false + + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + H/15 * * * * + false + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${github_release_tag} + + + false + + + + true + false + + 0 + false + + + + druid/druid-api/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/DataPipeline/config.xml index 9af36894a..17afc4067 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/DataPipeline/config.xml @@ -1,5 +1,5 @@ - + @@ -46,14 +46,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/Health/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ETLJobs/config.xml similarity index 61% rename from deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/Health/config.xml rename to deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ETLJobs/config.xml index 870f36491..5de7b91e0 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/Health/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/ETLJobs/config.xml @@ -1,13 +1,23 @@ - + false - + false false + + + + github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + + false + + + 0 0 @@ -18,25 +28,25 @@ - - + + 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/project-sunbird/sunbird-data-pipeline.git - ${branch_or_tag} + ${github_release_tag} false - pipelines/deploy/health-check/Jenkinsfile-dp - true + pipelines/build/adhoc-jobs/Jenkinsfile + false false diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Secor/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Secor/config.xml index c8cf0cb5d..35dfa6393 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Secor/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/Secor/config.xml @@ -46,7 +46,7 @@ - + H/15 * * * * false diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml index 5bd27bb90..01eeb5d51 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/KnowledgePlatform/config.xml @@ -1,5 +1,5 @@ - + @@ -57,8 +57,8 @@ - - + + 2 @@ -87,4 +87,4 @@ false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Neo4j/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Neo4j/config.xml index 73cfe087e..4dfbddd06 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Neo4j/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Neo4j/config.xml @@ -1,5 +1,5 @@ - + @@ -46,14 +46,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/SyncTool/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/SyncTool/config.xml index 28432d6de..09e72999b 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/SyncTool/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/SyncTool/config.xml @@ -1,5 +1,5 @@ - + @@ -46,14 +46,14 @@ - + H/15 * * * * false - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Yarn/config.xml index b04764d9b..b29e9d14e 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -58,8 +58,8 @@ - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/CollectionEditor/config.xml b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/CollectionEditor/config.xml index d5b251028..2851393b4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/CollectionEditor/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/CollectionEditor/config.xml @@ -1,5 +1,5 @@ - + false @@ -40,12 +40,12 @@ - - + + 2 - https://github.com/project-sunbird/sunbird-collection-editor.git + https://github.com/project-sunbird/sunbird-collection-editor diff --git a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentEditor/config.xml b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentEditor/config.xml index 819f456cb..64757a95e 100644 --- a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentEditor/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentEditor/config.xml @@ -1,5 +1,5 @@ - + false @@ -40,8 +40,8 @@ - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlayer/config.xml b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlayer/config.xml index 714b6d1ee..2c05092c2 100644 --- a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlayer/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlayer/config.xml @@ -1,5 +1,5 @@ - + false @@ -40,8 +40,8 @@ - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlugins/config.xml b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlugins/config.xml index 144527f9b..2c87891f4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlugins/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/ContentPlugins/config.xml @@ -1,5 +1,5 @@ - + false @@ -40,8 +40,8 @@ - - + + 2 diff --git a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/GenericEditor/config.xml b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/GenericEditor/config.xml index 25d012db1..8e12940dc 100644 --- a/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/GenericEditor/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Plugins/jobs/GenericEditor/config.xml @@ -1,5 +1,5 @@ - + false @@ -40,8 +40,8 @@ - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/config.xml index 6fe3ca0b6..ca27afff7 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/config.xml index 37e2c9b17..ca27afff7 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/config.xml @@ -3,7 +3,7 @@ - + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/APIManager/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/APIManager/config.xml index ac77776dc..04ef70b57 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/APIManager/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/APIManager/config.xml @@ -1,5 +1,5 @@ - + @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false @@ -63,13 +63,13 @@ return """<b>This parameter is not used</b>"&quo ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false @@ -99,6 +99,7 @@ return """<b>This parameter is not used</b>"&quo project false + 1 @@ -115,8 +116,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Assessment/config.xml new file mode 100644 index 000000000..1353dd4e7 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Assessment/config.xml @@ -0,0 +1,150 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/Assessment + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + Assessment + Deploy/dev/Core_disabled/Assessment + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + Assessment + Deploy/dev/Core_disabled/Assessment + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/Assessment + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/assessment/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Badger/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Badger/config.xml index 2a243f37d..139322e5f 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Badger/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Badger/config.xml @@ -1,7 +1,7 @@ - + - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -20,7 +20,7 @@ - + false false @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false Badger + Deploy/dev/Core/Badger ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Badger + Deploy/dev/Core/Badger ET_FORMATTED_HTML @@ -104,8 +106,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -134,4 +136,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Bootstrap/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Bootstrap/config.xml new file mode 100644 index 000000000..54d79c38f --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Bootstrap/config.xml @@ -0,0 +1,112 @@ + + + + + false + + + + -1 + -1 + -1 + 5 + + + + false + false + + + + + hosts + <font color=dimgray size=2><b>Enter the host name to be bootstrapped.</b></font> + all + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + Bootstrap + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + Bootstrap + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + pipelines/bootstrap/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CassandraTrigger/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CassandraTrigger/config.xml index 3f345abec..9fb0436c4 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CassandraTrigger/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CassandraTrigger/config.xml @@ -1,5 +1,5 @@ - + false @@ -13,7 +13,7 @@ - + false false @@ -25,39 +25,40 @@ ArtifactUpload/dev/Core/CassandraTrigger false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-9600649228560 1 false - + false CassandraTrigger + Deploy/dev/Core/CassandraTrigger PT_SINGLE_SELECT false 1 - + build_number choice-parameter-9600651313765 1 false - + false CassandraTrigger + Deploy/dev/Core/CassandraTrigger artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-9600653373369 1 false - + false CassandraTrigger + Deploy/dev/Core/CassandraTrigger artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false CassandraTrigger + Deploy/dev/Core/CassandraTrigger ET_FORMATTED_HTML true - - branch_or_tag + + public_branch choice-parameter-2620434998790477 1 false - + false CassandraTrigger + Deploy/dev/Core/CassandraTrigger ET_FORMATTED_HTML @@ -180,12 +185,12 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/project-sunbird/sunbird-learning-platform.git @@ -210,4 +215,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cert/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cert/config.xml new file mode 100644 index 000000000..5713bdacb --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cert/config.xml @@ -0,0 +1,150 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/Cert + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + Cert + Deploy/dev/Core/Cert + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + Cert + Deploy/dev/Core/Cert + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/Cert + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/cert/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml new file mode 100644 index 000000000..44e278d0d --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/CertRegistryService/config.xml @@ -0,0 +1,148 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/CertRegistryService + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + CertRegistryService + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + CertRegistryService + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/CertRegistryService + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/cert-registry-service/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Content/config.xml index b71b638f5..c7253c951 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Content/config.xml @@ -1,7 +1,7 @@ - + - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -20,7 +20,7 @@ - + false false @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false Content + Deploy/dev/Core/Content ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Content + Deploy/dev/Core/Content ET_FORMATTED_HTML @@ -115,8 +117,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -135,7 +137,7 @@ return """<b>This parameter is not used</b>"&quo true false - 0 + 1 false @@ -145,4 +147,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/EncService/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/EncService/config.xml new file mode 100644 index 000000000..46fec0d7d --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/EncService/config.xml @@ -0,0 +1,148 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/EncService + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + EncService + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + EncService + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/EncService + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/enc/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml new file mode 100644 index 000000000..c355d4b94 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/KnowledgeMW/config.xml @@ -0,0 +1,150 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/KnowledgeMW + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + KnowledgeMW + Deploy/dev/Core/KnowledgeMW + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + KnowledgeMW + Deploy/dev/Core/KnowledgeMW + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/KnowledgeMW + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/knowledge-mw-service/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Learner/config.xml index 3ee98f84f..db227c473 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Learner/config.xml @@ -1,5 +1,5 @@ - + @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false Learner + Deploy/dev/Core/Learner ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Learner + Deploy/dev/Core/Learner ET_FORMATTED_HTML @@ -115,8 +117,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Lms/config.xml index ee332f9bc..580fb9c89 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Lms/config.xml @@ -1,5 +1,5 @@ - + @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false Lms + Deploy/dev/Core/Lms ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Lms + Deploy/dev/Core/Lms ET_FORMATTED_HTML @@ -115,8 +117,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Logging/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Logging/config.xml index 57b6a7530..70c06dbdc 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Logging/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Logging/config.xml @@ -1,5 +1,5 @@ - + false @@ -19,13 +19,13 @@ - + private_branch choice-parameter-186759255678992 1 false - + false Logging + Deploy/dev/Core/Logging ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-186759262347032 1 false - + false Logging + Deploy/dev/Core/Logging ET_FORMATTED_HTML @@ -97,8 +99,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Monitoring/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Monitoring/config.xml index be515d6a9..5d8bcbb80 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Monitoring/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Monitoring/config.xml @@ -1,5 +1,5 @@ - + false @@ -19,13 +19,13 @@ - + private_branch choice-parameter-186775018293883 1 false - + false Monitoring + Deploy/dev/Core/Monitoring ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-186775024902722 1 false - + false Monitoring + Deploy/dev/Core/Monitoring ET_FORMATTED_HTML @@ -100,8 +102,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Notification/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Notification/config.xml new file mode 100644 index 000000000..ccf3d0f4a --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Notification/config.xml @@ -0,0 +1,148 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/Notification + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + Notification + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + Notification + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/Notification + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/notification/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardAPIs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardAPIs/config.xml index 9676eb0fe..7e9662ce0 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardAPIs/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardAPIs/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-117346587048120 1 false - + false OnboardAPIs + Deploy/dev/Core/OnboardAPIs ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-117346594984751 1 false - + false OnboardAPIs + Deploy/dev/Core/OnboardAPIs ET_FORMATTED_HTML @@ -89,12 +91,12 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/project-sunbird/sunbird-devops diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardConsumers/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardConsumers/config.xml index 238dd309e..d1a4d4cfc 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardConsumers/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/OnboardConsumers/config.xml @@ -1,5 +1,5 @@ - + false @@ -13,19 +13,19 @@ - + false false - + private_branch choice-parameter-186788940114941 1 false - + false OnboardConsumers + Deploy/dev/Core/OnboardConsumers ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-186788943531283 1 false - + false OnboardConsumers + Deploy/dev/Core/OnboardConsumers ET_FORMATTED_HTML @@ -82,8 +84,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/PlayerCDN/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/PlayerCDN/config.xml index a4b38cda3..c7d437a52 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/PlayerCDN/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/PlayerCDN/config.xml @@ -1,5 +1,5 @@ - + @@ -38,13 +38,13 @@ false - + private_branch choice-parameter-2544395024638227 1 false - + false PlayerCDN + Deploy/dev/Core/PlayerCDN ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false PlayerCDN + Deploy/dev/Core/PlayerCDN ET_FORMATTED_HTML @@ -121,8 +123,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Print/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Print/config.xml new file mode 100644 index 000000000..d6239bef9 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Print/config.xml @@ -0,0 +1,150 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/Print + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + Print + Deploy/dev/Core/Print + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + Print + Deploy/dev/Core/Print + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/Print + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/print-service/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ReloadContainerServices/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ReloadContainerServices/config.xml index bc7f06688..3e98e5237 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ReloadContainerServices/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ReloadContainerServices/config.xml @@ -1,5 +1,5 @@ - + false @@ -39,8 +39,8 @@ - - + + 2 @@ -65,7 +65,7 @@ pipelines/reloadConfigurations/Jenkinsfile - true + false false diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/TelemetryLogstash/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/TelemetryLogstash/config.xml new file mode 100644 index 000000000..3f59f70e3 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/TelemetryLogstash/config.xml @@ -0,0 +1,115 @@ + + + + Deploys Telemetry Logstash Service + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-186827048551040 + 1 + + false + + + + false + + + TelemetryLogstash + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-186827051364357 + 1 + + false + + + + false + + + TelemetryLogstash + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/telemetry-logstash/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/UserOrg/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/UserOrg/config.xml new file mode 100644 index 000000000..557e5f459 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/UserOrg/config.xml @@ -0,0 +1,148 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/UserOrg + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + UserOrg + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + UserOrg + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/Core/UserOrg + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 1 + false + + + + pipelines/deploy/userorg/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/config.xml index 37e2c9b17..ca27afff7 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/config.xml @@ -3,7 +3,7 @@ - + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsAPI/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsAPI/config.xml index bbce94b85..dfe786f73 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsAPI/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsAPI/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/DataPipeline/Analytics false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-9061667106947 1 false - + false AnalyticsAPI + Deploy/dev/DataPipeline/AnalyticsAPI PT_SINGLE_SELECT false 1 - + build_number choice-parameter-9061670571657 1 false - + false AnalyticsAPI + Deploy/dev/DataPipeline/AnalyticsAPI artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-9061673103864 1 false - + false AnalyticsAPI + Deploy/dev/DataPipeline/AnalyticsAPI artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsAPI + Deploy/dev/DataPipeline/AnalyticsAPI ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsAPI + Deploy/dev/DataPipeline/AnalyticsAPI ET_FORMATTED_HTML @@ -177,8 +182,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsFetchLogs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsFetchLogs/config.xml index 110022cbc..a8ca2905d 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsFetchLogs/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsFetchLogs/config.xml @@ -1,5 +1,5 @@ - + @@ -48,13 +48,13 @@ - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsFetchLogs + Deploy/dev/DataPipeline/AnalyticsFetchLogs ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsFetchLogs + Deploy/dev/DataPipeline/AnalyticsFetchLogs ET_FORMATTED_HTML @@ -111,8 +113,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsGeoLocationDBSetup/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsGeoLocationDBSetup/config.xml index 3a4a7183b..23e99c74f 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsGeoLocationDBSetup/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsGeoLocationDBSetup/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-188130479612620 1 false - + false AnalyticsGeoLocationDBSetup + Deploy/dev/DataPipeline/AnalyticsGeoLocationDBSetup ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-188130482273762 1 false - + false AnalyticsGeoLocationDBSetup + Deploy/dev/DataPipeline/AnalyticsGeoLocationDBSetup ET_FORMATTED_HTML @@ -89,8 +91,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsPopulatePSQLConsumerChannelMapping/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsPopulatePSQLConsumerChannelMapping/config.xml index 1303af19c..1f1597299 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsPopulatePSQLConsumerChannelMapping/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsPopulatePSQLConsumerChannelMapping/config.xml @@ -1,5 +1,5 @@ - + @@ -20,7 +20,7 @@ - + false false @@ -54,13 +54,13 @@ analytics-team false - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsPopulatePSQLConsumerChannelMapping + Deploy/dev/DataPipeline/AnalyticsPopulatePSQLConsumerChannelMapping ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsPopulatePSQLConsumerChannelMapping + Deploy/dev/DataPipeline/AnalyticsPopulatePSQLConsumerChannelMapping ET_FORMATTED_HTML @@ -115,11 +117,10 @@ return """<b>This parameter is not used</b>"&quo project false - 1 - - + + 2 @@ -148,4 +149,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsReplayJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsReplayJobs/config.xml index adcdeca5a..cf23738d9 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsReplayJobs/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsReplayJobs/config.xml @@ -1,5 +1,5 @@ - + @@ -68,13 +68,13 @@ 30 false - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsReplayJobs + Deploy/dev/DataPipeline/AnalyticsReplayJobs ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsReplayJobs + Deploy/dev/DataPipeline/AnalyticsReplayJobs ET_FORMATTED_HTML @@ -131,8 +133,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsStartJobManager/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsStartJobManager/config.xml index 62cacabb9..99fa738f0 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsStartJobManager/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsStartJobManager/config.xml @@ -1,5 +1,5 @@ - + @@ -20,7 +20,7 @@ - + false false @@ -32,13 +32,13 @@ 5 false - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsStartJobManager + Deploy/dev/DataPipeline/AnalyticsStartJobManager ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsStartJobManager + Deploy/dev/DataPipeline/AnalyticsStartJobManager ET_FORMATTED_HTML @@ -93,11 +95,10 @@ return """<b>This parameter is not used</b>"&quo project false - 1 - - + + 2 @@ -126,4 +127,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSubmitJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSubmitJobs/config.xml index 639011b4c..e3639ec0f 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSubmitJobs/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSubmitJobs/config.xml @@ -1,5 +1,5 @@ - + @@ -20,7 +20,7 @@ - + false false @@ -32,13 +32,13 @@ 5 false - + private_branch choice-parameter-2544395024638227 1 false - + false AnalyticsSubmitJobs + Deploy/dev/DataPipeline/AnalyticsSubmitJobs ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false AnalyticsSubmitJobs + Deploy/dev/DataPipeline/AnalyticsSubmitJobs ET_FORMATTED_HTML @@ -86,16 +88,17 @@ return """<b>This parameter is not used</b>"&quo + 0 + 0 false project false - 1 - - + + 2 @@ -124,4 +127,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml new file mode 100644 index 000000000..940c48fa6 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ApiModule/config.xml @@ -0,0 +1,178 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/DataPipeline/ApiModule + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + build_number + + choice-parameter-9061670571657 + 1 + + false + + + + false + + + ApiModule + Deploy/dev/DataPipeline/ApiModule + + artifact_source + ET_FORMATTED_HTML + true + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + ApiModule + Deploy/dev/DataPipeline/ApiModule + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + ApiModule + Deploy/dev/DataPipeline/ApiModule + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/DataPipeline/ApiModule + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/druid-proxy/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataProducts/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataProducts/config.xml index 12d476f34..7826dc016 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataProducts/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataProducts/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/DataPipeline/Analytics false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-9061667106947 1 false - + false DataProducts + Deploy/dev/DataPipeline/DataProducts PT_SINGLE_SELECT false 1 - + build_number choice-parameter-9061670571657 1 false - + false DataProducts + Deploy/dev/DataPipeline/DataProducts artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-9061673103864 1 false - + false DataProducts + Deploy/dev/DataPipeline/DataProducts artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false DataProducts + Deploy/dev/DataPipeline/DataProducts ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false DataProducts + Deploy/dev/DataPipeline/DataProducts ET_FORMATTED_HTML @@ -188,8 +193,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataproductsConfigUpdate/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataproductsConfigUpdate/config.xml index 08a817f99..6e006a10c 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataproductsConfigUpdate/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DataproductsConfigUpdate/config.xml @@ -1,5 +1,5 @@ - + @@ -35,13 +35,13 @@ Please provide the tag name. false - + private_branch choice-parameter-2544395024638227 1 false - + false DataproductsConfigUpdate + Deploy/dev/DataPipeline/DataproductsConfigUpdate ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false DataproductsConfigUpdate + Deploy/dev/DataPipeline/DataproductsConfigUpdate ET_FORMATTED_HTML @@ -101,8 +103,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DruidIngestion/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DruidIngestion/config.xml new file mode 100644 index 000000000..30f4f7c78 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/DruidIngestion/config.xml @@ -0,0 +1,138 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + action + <font color=red size=2><b> +To start task: [start-task] +To stop task: [stop-task] +</b></font> + + + start-task + stop-task + + + + + private_branch + + choice-parameter-193684482193674 + 1 + + false + + + + false + + + DruidIngestion + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-193684493817846 + 1 + + false + + + + false + + + DruidIngestion + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/ops/druid/Jenkinsfile.ingestion + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Keycloak/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLDruidContentIndexer/config.xml similarity index 78% rename from deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Keycloak/config.xml rename to deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLDruidContentIndexer/config.xml index 54b2dacce..d6bcdef93 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Keycloak/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLDruidContentIndexer/config.xml @@ -1,6 +1,13 @@ - + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + false @@ -22,14 +29,14 @@ absolute_job_path <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> - ArtifactUpload/dev/Core/Keycloak + Deploy/dev/DataPipeline/ETLJobs false artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> - choice-parameter-9600649228560 + choice-parameter-9061667106947 1 - Keycloak + ETLDruidContentIndexer PT_SINGLE_SELECT @@ -54,7 +61,7 @@ ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrif build_number - choice-parameter-9600651313765 + choice-parameter-9061670571657 1 - Keycloak + ETLDruidContentIndexer artifact_source ET_FORMATTED_HTML @@ -81,7 +88,7 @@ return "<b>Not Applicable</b>" artifact_version - choice-parameter-9600653373369 + choice-parameter-9061673103864 1 - Keycloak + ETLDruidContentIndexer artifact_source ET_FORMATTED_HTML @@ -123,12 +130,22 @@ return """<b>This parameter is not used</b>"&quo false - Keycloak + ETLDruidContentIndexer ET_FORMATTED_HTML true + + script_to_run + <font color=dimgray size=2><b> +Default Druid content indexer job will run as cron.</b></font> + + + DRUID_CONTENT_INDEXER + + + branch_or_tag @@ -137,7 +154,7 @@ return """<b>This parameter is not used</b>"&quo - Keycloak + ETLDruidContentIndexer ET_FORMATTED_HTML true - - sunbird_auth_branch_or_tag - <font color=dimgray size=2><b>Provide the branch or tag of sunbird auth repo.</b></font> - - false - + + invoke_type + <font color=dimgray size=2><b> +If you select the "execute script from the invoke_type drop down then choose which script you want to execute."</b></font> + + + execute-script + + + @@ -174,16 +195,9 @@ return """<b>This parameter is not used</b>"&quo - - - ArtifactUpload/dev/Core/Keycloak - - SUCCESS - 0 - BLUE - true - - + + H 22 * * * + @@ -192,7 +206,7 @@ return """<b>This parameter is not used</b>"&quo 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/project-sunbird/sunbird-data-pipeline.git @@ -202,10 +216,18 @@ return """<b>This parameter is not used</b>"&quo false - + + + true + false + + 0 + false + + - pipelines/deploy/keycloak/Jenkinsfile - true + pipelines/deploy/content-snapshot-indexer/Jenkinsfile + false false diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml new file mode 100644 index 000000000..f559a8b30 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLJobs/config.xml @@ -0,0 +1,245 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/DataPipeline/ETLJobs + false + + + invoke_type + <font color=dimgray size=2><b> +Please select the action deploy or execute script.</b></font> + + + deploy + execute-script + + + + + script_to_run + <font color=dimgray size=2><b> +If you select the "execute script from the invoke_type drop down then choose which script you want to execute."</b></font> + + + USER_CACHE_INDEXER + DRUID_CONTENT_INDEXER + CONTENT_CACHE_INDEXER + DIALCODE_CACHE_INDEXER + + + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> + choice-parameter-9061667106947 + 1 + + false + + + + false + + + ETLJobs + + + PT_SINGLE_SELECT + false + 1 + + + build_number + + choice-parameter-9061670571657 + 1 + + false + + + + false + + + ETLJobs + + artifact_source + ET_FORMATTED_HTML + true + + + artifact_version + + choice-parameter-9061673103864 + 1 + + false + + + + false + + + ETLJobs + + artifact_source + ET_FORMATTED_HTML + true + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + ETLJobs + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + ETLJobs + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + ArtifactUpload/dev/DataPipeline/ETLJobs + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/content-snapshot-indexer/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cassandra/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLUserCacheIndexer/config.xml similarity index 81% rename from deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cassandra/config.xml rename to deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLUserCacheIndexer/config.xml index 4642fe0f4..553749d1f 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/Cassandra/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/ETLUserCacheIndexer/config.xml @@ -1,6 +1,13 @@ - + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + false @@ -22,14 +29,14 @@ absolute_job_path <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> - ArtifactUpload/dev/Core/Cassandra + Deploy/dev/DataPipeline/ETLJobs false artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> - choice-parameter-9600649228560 + choice-parameter-9061667106947 1 - Cassandra + ETLUserCacheIndexer PT_SINGLE_SELECT @@ -54,7 +61,7 @@ ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrif build_number - choice-parameter-9600651313765 + choice-parameter-9061670571657 1 - Cassandra + ETLUserCacheIndexer artifact_source ET_FORMATTED_HTML @@ -81,7 +88,7 @@ return "<b>Not Applicable</b>" artifact_version - choice-parameter-9600653373369 + choice-parameter-9061673103864 1 - Cassandra + ETLUserCacheIndexer artifact_source ET_FORMATTED_HTML @@ -123,12 +130,22 @@ return """<b>This parameter is not used</b>"&quo false - Cassandra + ETLUserCacheIndexer ET_FORMATTED_HTML true + + script_to_run + <font color=dimgray size=2><b> +user cache indexer script will run.</b></font> + + + USER_CACHE_INDEXER + + + branch_or_tag @@ -137,7 +154,7 @@ return """<b>This parameter is not used</b>"&quo - Cassandra + ETLUserCacheIndexer ET_FORMATTED_HTML true + + invoke_type + <font color=dimgray size=2><b> +Please select the action deploy or execute script.</b></font> + + + execute-script + + + @@ -168,16 +195,9 @@ return """<b>This parameter is not used</b>"&quo - - - ArtifactUpload/dev/Core/Cassandra - - SUCCESS - 0 - BLUE - true - - + + H 22 * * * + @@ -186,7 +206,7 @@ return """<b>This parameter is not used</b>"&quo 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/project-sunbird/sunbird-data-pipeline.git @@ -206,7 +226,7 @@ return """<b>This parameter is not used</b>"&quo - pipelines/deploy/cassandra-deploy/Jenkinsfile + pipelines/deploy/content-snapshot-indexer/Jenkinsfile false diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaIndexer/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaIndexer/config.xml index 4a67ac124..906b7afeb 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaIndexer/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaIndexer/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-188153004575082 1 false - + false KafkaIndexer + Deploy/dev/DataPipeline/KafkaIndexer ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-188153015326365 1 false - + false KafkaIndexer + Deploy/dev/DataPipeline/KafkaIndexer ET_FORMATTED_HTML @@ -89,8 +91,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaSetup/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaSetup/config.xml index bb134aaf6..6a96ea641 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaSetup/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/KafkaSetup/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-188157054642539 1 false - + false KafkaSetup + Deploy/dev/DataPipeline/KafkaSetup ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-188157057910170 1 false - + false KafkaSetup + Deploy/dev/DataPipeline/KafkaSetup ET_UNORDERED_LIST @@ -89,8 +91,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Monitoring/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Monitoring/config.xml index b97a6b518..dcfdb361b 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Monitoring/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Monitoring/config.xml @@ -101,7 +101,7 @@ return """<b>This parameter is not used</b>"&quo 2 - https://github.com/project-sunbird/sunbird-data-pipeline.git + https://github.com/project-sunbird/sunbird-devops.git diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml index 2f4da21a1..293eb390e 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Secor/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/DataPipeline/Secor false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-9061667106947 1 false - + false Secor + Deploy/dev/DataPipeline/Secor PT_SINGLE_SELECT false 1 - + build_number choice-parameter-9061670571657 1 false - + false Secor + Deploy/dev/DataPipeline/Secor artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-9061673103864 1 false - + false Secor + Deploy/dev/DataPipeline/Secor artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false Secor + Deploy/dev/DataPipeline/Secor ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Secor + Deploy/dev/DataPipeline/Secor ET_FORMATTED_HTML @@ -188,8 +193,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml index 9f51edc8f..7a1bcd77f 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/DataPipeline/DataPipeline false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-9061667106947 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn PT_SINGLE_SELECT false 1 - + build_number choice-parameter-9061670571657 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-9061673103864 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn ET_FORMATTED_HTML true - + job_names_to_deploy <font color=green size=2><b>Choose the samza app names to deploy and this will kill the selected samza app and deploy them. Multi selection is available.</b></font> choice-parameter-8026182213525059 1 false - + false Yarn + Deploy/dev/DataPipeline/Yarn PT_MULTI_SELECT @@ -210,8 +218,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/ESMapping/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/ESMapping/config.xml new file mode 100644 index 000000000..415af374c --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/ESMapping/config.xml @@ -0,0 +1,147 @@ + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + branch_or_tag + + choice-parameter-9675060449018219 + 1 + + false + + + + false + + + ESMapping + Deploy/dev/KnowledgePlatform/ESMapping + + + ET_FORMATTED_HTML + true + + + private_branch + + choice-parameter-9675060450520341 + 1 + + false + + + + false + + + ESMapping + Deploy/dev/KnowledgePlatform/ESMapping + + + ET_FORMATTED_HTML + true + + + file_name + <font color=dimgray size=2><b>enter the file name for template or mapping<br> + please add name without json extension.<br> + <br> + ex: test.json (templates or mapping) is added in ansible files, <br> + input should be test <br> + <br> + comma seperated values for multiple files <br> + ex: filename1,filename22 <br> + <br> + enter all to run all template and mappings <br> + </b></font> + + false + + + tag + + + + template + mapping + all + + + + + + + 0 + 0 + + false + project + false + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-learning-platform.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/project-sunbird/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Learning/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Learning/config.xml index 4734ae910..a5af84b20 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Learning/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Learning/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/KnowledgePlatform/KnowledgePlatform false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-12566219493457 1 false - + false Learning + Deploy/dev/KnowledgePlatform/Learning PT_SINGLE_SELECT false 1 - + build_number choice-parameter-12566221440336 1 false - + false Learning + Deploy/dev/KnowledgePlatform/Learning artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-12566223395914 1 false - + false Learning + Deploy/dev/KnowledgePlatform/Learning artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false Learning + Deploy/dev/KnowledgePlatform/Learning ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Learning + Deploy/dev/KnowledgePlatform/Learning ET_FORMATTED_HTML @@ -188,8 +193,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Monitoring/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Monitoring/config.xml index 5dc708d76..a12d3f078 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Monitoring/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Monitoring/config.xml @@ -101,7 +101,7 @@ return """<b>This parameter is not used</b>"&quo 2 - https://github.com/project-sunbird/sunbird-learning-platform.git + https://github.com/project-sunbird/sunbird-devops.git diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jDefinitionUpdate/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jDefinitionUpdate/config.xml index bd7b405d6..b2b030f66 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jDefinitionUpdate/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jDefinitionUpdate/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-188174424977419 1 false - + false Neo4jDefinitionUpdate + Deploy/dev/KnowledgePlatform/Neo4jDefinitionUpdate ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-188174429374717 1 false - + false Neo4jDefinitionUpdate + Deploy/dev/KnowledgePlatform/Neo4jDefinitionUpdate ET_FORMATTED_HTML @@ -92,8 +94,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Search/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Search/config.xml index 1981e866f..434510f8e 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Search/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Search/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/KnowledgePlatform/KnowledgePlatform false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-14760318274338 1 false - + false Search + Deploy/dev/KnowledgePlatform/Search PT_SINGLE_SELECT false 1 - + build_number choice-parameter-14760320375443 1 false - + false Search + Deploy/dev/KnowledgePlatform/Search artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-14760322347747 1 false - + false Search + Deploy/dev/KnowledgePlatform/Search artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false Search + Deploy/dev/KnowledgePlatform/Search ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Search + Deploy/dev/KnowledgePlatform/Search ET_FORMATTED_HTML @@ -188,8 +193,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml index abeeb8955..af5b7e4c3 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -32,39 +32,40 @@ ArtifactUpload/dev/KnowledgePlatform/Yarn false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-2874232768088857 1 false - + false Yarn + Deploy/dev/KnowledgePlatform/Yarn PT_SINGLE_SELECT false 1 - + build_number choice-parameter-2874449088885218 1 false - + false Yarn + Deploy/dev/KnowledgePlatform/Yarn artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-2877900313772031 1 false - + false Yarn + Deploy/dev/KnowledgePlatform/Yarn artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false Yarn + Deploy/dev/KnowledgePlatform/Yarn ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false Yarn + Deploy/dev/KnowledgePlatform/Yarn ET_FORMATTED_HTML @@ -188,8 +193,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/ContentPlayer/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/ContentPlayer/config.xml index d1097f1bd..999dde5c3 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/ContentPlayer/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/ContentPlayer/config.xml @@ -1,11 +1,11 @@ - + false - + false false @@ -17,39 +17,40 @@ ArtifactUpload/dev/Plugins/ContentPlayer false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-620062284617427 1 false - + false ContentPlayer + Deploy/dev/Plugins/ContentPlayer PT_SINGLE_SELECT false 1 - + build_number choice-parameter-620062286335119 1 false - + false ContentPlayer + Deploy/dev/Plugins/ContentPlayer artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-620062288387509 1 false - + false ContentPlayer + Deploy/dev/Plugins/ContentPlayer artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false ContentPlayer + Deploy/dev/Plugins/ContentPlayer ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false ContentPlayer + Deploy/dev/Plugins/ContentPlayer ET_FORMATTED_HTML @@ -172,8 +177,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/GenericEditor/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/GenericEditor/config.xml index 0890c7d1f..c4a6262e7 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/GenericEditor/config.xml +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Plugins/jobs/GenericEditor/config.xml @@ -1,11 +1,11 @@ - + false - + false false @@ -17,39 +17,40 @@ ArtifactUpload/dev/Plugins/GenericEditor false - + artifact_source <font color=dimgray size=2><b> ArtifactRepo - Download the artifact from azure blob, JenkinsJob - Use the atrifact from Jenkins job.</b></font> choice-parameter-620062284617427 1 false - + false GenericEditor + Deploy/dev/Plugins/GenericEditor PT_SINGLE_SELECT false 1 - + build_number choice-parameter-620062286335119 1 false - + false GenericEditor + Deploy/dev/Plugins/GenericEditor artifact_source ET_FORMATTED_HTML true - + artifact_version choice-parameter-620062288387509 1 false - + false GenericEditor + Deploy/dev/Plugins/GenericEditor artifact_source ET_FORMATTED_HTML true - + private_branch choice-parameter-2544395024638227 1 false - + false GenericEditor + Deploy/dev/Plugins/GenericEditor ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2620434998790477 1 false - + false GenericEditor + Deploy/dev/Plugins/GenericEditor ET_FORMATTED_HTML @@ -172,8 +177,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/config.xml new file mode 100644 index 000000000..68477b8ac --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/config.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + All + false + false + + + + + + + + false + + + + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/jobs/DeployedVersions/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/jobs/DeployedVersions/config.xml new file mode 100644 index 000000000..2c667ec58 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Summary/jobs/DeployedVersions/config.xml @@ -0,0 +1,74 @@ + + + + + false + + + + -1 + -1 + -1 + 10 + + + + + false + false + + + 0 + 0 + + false + project + false + + + + + + + Deploy/dev/Core/Content,Deploy/dev/Core/Player,Deploy/dev/Core/Learner,Deploy/dev/Core/Telemetry,Deploy/dev/Core/Proxy,Deploy/dev/KnowledgePlatform/Search,Deploy/dev/KnowledgePlatform/Learning,Deploy/dev/KnowledgePlatform/Yarn,Deploy/dev/DataPipeline/Secor,Deploy/dev/DataPipeline/Yarn,Deploy/dev/DataPipeline/AnalyticsAPI,Deploy/dev/DataPipeline/DataProducts + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/summary/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/config.xml index 5be56a4e0..ca27afff7 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchBackup/config.xml index cf408df53..54c8ced7a 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 2 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml index 30252b1ea..0f8f73f2d 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ApplicationElasticsearchRestore/config.xml @@ -27,7 +27,7 @@ @@ -124,4 +124,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/CassandraBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/CassandraBackup/config.xml index a1a5cfe29..9ceaefbe6 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/CassandraBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/CassandraBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 3 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearDockerImages/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearDockerImages/config.xml index 27be9187b..0abf0e9bf 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearDockerImages/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearDockerImages/config.xml @@ -25,19 +25,16 @@ false - + H 2 * * * false - echo "Before clearing" -df -h -docker image rm $(docker images | awk 'NR>1 {print $3}') --force 2> /dev/null -echo "After clearing" -df -h + #docker image rm $(docker images | awk 'NR>1 {print $3}') --force 2> /dev/null +docker system prune -af - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearWorkspace/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearWorkspace/config.xml index 54a50a49d..087d526b0 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearWorkspace/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ClearWorkspace/config.xml @@ -1,22 +1,18 @@ - false - + false false - 0 - 0 false project false - 1 @@ -26,25 +22,19 @@ false - + H 4 * * * false - echo "Before clearing" -df -h - -rm -rf /var/lib/jenkins/workspace/Build + rm -rf /var/lib/jenkins/workspace/Build rm -rf /var/lib/jenkins/workspace/Provision -rm -rf /var/lib/jenkins/workspace/ArtifactUpload -rm -rf /var/lib/jenkins/workspace/Deploy rm -rf /var/lib/jenkins/workspace/OpsAdministration - -echo "After clearing" -df -h +rm -rf /var/lib/jenkins/workspace/ArtifactUpload +rm -rf /var/lib/jenkins/workspace/Deploy - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GrafanaBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GrafanaBackup/config.xml index 19008ae57..68aafba32 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GrafanaBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/GrafanaBackup/config.xml @@ -94,7 +94,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/JenkinsBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/JenkinsBackup/config.xml index 418d6becd..b09d7e542 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/JenkinsBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/JenkinsBackup/config.xml @@ -83,7 +83,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 4 * * * @@ -98,7 +98,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/KongMigration/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/KongMigration/config.xml new file mode 100644 index 000000000..6d1c763ae --- /dev/null +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/KongMigration/config.xml @@ -0,0 +1,137 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/Core/APIManager + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + private_branch + + choice-parameter-2544395024638227 + 1 + + false + + + + false + + + KongMigration + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + false + + + + false + + + KongMigration + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/ops/kong-migration/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/LoggerElasticSearchBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/LoggerElasticSearchBackup/config.xml index 07e68fe7e..17fa0572d 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/LoggerElasticSearchBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/LoggerElasticSearchBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 1 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresManagedBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ManagedPostgresBackup/config.xml similarity index 89% rename from deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresManagedBackup/config.xml rename to deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ManagedPostgresBackup/config.xml index 1f13a4b49..51657f25f 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresManagedBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/ManagedPostgresBackup/config.xml @@ -1,5 +1,5 @@ - + false @@ -19,13 +19,13 @@ - + private_branch choice-parameter-189733897665665 1 false - + false PostgresManagedBackup + OpsAdministration/dev/Core/PostgresManagedBackup ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-189733901134743 1 false - + false PostgresManagedBackup + OpsAdministration/dev/Core/PostgresManagedBackup ET_FORMATTED_HTML @@ -90,13 +92,13 @@ return """<b>This parameter is not used</b>"&quo - + H 3 * * * - - + + 2 @@ -105,7 +107,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlBackup/config.xml index 17dafade9..c8d49c9cd 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + * 12 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false @@ -118,5 +118,5 @@ return """<b>This parameter is not used</b>"&quo false - false - + true + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlRestore/config.xml index 64500d857..994908853 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlRestore/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PostgresqlRestore/config.xml @@ -90,7 +90,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusBackup/config.xml index 2f40bf26b..62420abc2 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 1 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusSanpshot/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusSanpshot/config.xml index e305739a8..497f1633b 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusSanpshot/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/PrometheusSanpshot/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 2 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false @@ -119,4 +119,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/config.xml index 5be56a4e0..ca27afff7 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/CassandraBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/CassandraBackup/config.xml index eac80e829..e99463288 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/CassandraBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/CassandraBackup/config.xml @@ -103,7 +103,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 5 * * * @@ -118,7 +118,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ConsumerLagAlert/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ConsumerLagAlert/config.xml index 99b502af9..78c54d89f 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ConsumerLagAlert/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ConsumerLagAlert/config.xml @@ -97,7 +97,7 @@ return """<b>This parameter is not used</b>"&quo - + H/60 * * * * diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DataproductsAlert/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DataproductsAlert/config.xml index 36c5c0049..570cefb6e 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DataproductsAlert/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DataproductsAlert/config.xml @@ -91,7 +91,7 @@ return """<b>This parameter is not used</b>"&quo - + 30 10 * * * @@ -106,7 +106,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false @@ -126,4 +126,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxdbRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DruidPostgresBackup/config.xml similarity index 54% rename from deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxdbRestore/config.xml rename to deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DruidPostgresBackup/config.xml index 47ef53cdc..b63f0b4eb 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxdbRestore/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/DruidPostgresBackup/config.xml @@ -4,28 +4,25 @@ false + + + -1 + -1 + -1 + 5 + + + false false - - influxdb_host - Influxdb ip or hostname where data needs to be restored. - - false - - - influxdb_restore_file_name - influxdb backup file name which needs to be restored. - - false - private_branch - choice-parameter-4812402663506673 + choice-parameter-193687699799775 1 - Influxdb_restore + DruidPostgresBackup ET_FORMATTED_HTML - false + true + + branch_or_tag + + choice-parameter-193687712890752 + 1 + + false + + + + false + + + DruidPostgresBackup + + + ET_FORMATTED_HTML + true + + + remote + + druid-postgres + false + @@ -74,10 +103,18 @@ return """<b>This parameter is not used</b>"&quo false - + + + true + false + + 0 + false + + - pipelines/backup/influxdb-restore/Jenkinsfile - true + pipelines/backup/postgresql-backup/Jenkinsfile + false false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchBackup/config.xml index a5dbaf9ec..068045858 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchBackup/config.xml @@ -97,7 +97,7 @@ return """<b>This parameter is not used</b>"&quo - + H 2 * * * @@ -112,7 +112,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml index bfa6996b0..26d888332 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/ElasticsearchRestore/config.xml @@ -111,7 +111,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxDBBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxDBBackup/config.xml index 61ae7e1d5..14d635555 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxDBBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/InfluxDBBackup/config.xml @@ -91,7 +91,7 @@ return """<b>This parameter is not used</b>"&quo - + 0 0 * * * @@ -106,7 +106,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/MonitDeploy/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/MonitDeploy/config.xml new file mode 100644 index 000000000..abbec5a20 --- /dev/null +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/MonitDeploy/config.xml @@ -0,0 +1,138 @@ + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-193690420291616 + 1 + + false + + + + false + + + MonitDeploy + OpsAdministration/dev/DataPipeline/MonitDeploy + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-193690422104853 + 1 + + false + + + + false + + + MonitDeploy + OpsAdministration/dev/DataPipeline/MonitDeploy + + + ET_FORMATTED_HTML + true + + + tag + + + + all + logstash + secor + yarnmaster + yarnslave + influxdb + cassandra + analytics-api + redis + zookeeper + + + + + + + 0 + 0 + + false + project + false + + + + + + + + + 2 + + + https://github.com/kaali09/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/deploy/monit/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/PostgresqlBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/PostgresqlBackup/config.xml index 24282ed99..fac30b936 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/PostgresqlBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/PostgresqlBackup/config.xml @@ -90,7 +90,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisDumpMonitoring/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisDumpMonitoring/config.xml new file mode 100644 index 000000000..d7cb16b5a --- /dev/null +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisDumpMonitoring/config.xml @@ -0,0 +1,127 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-2914866095851737 + 1 + + false + + + + false + + + RedisDumpMonitoring + OpsAdministration/dev/DataPipeline/RedisDumpMonitoring + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2914866097410763 + 1 + + false + + + + false + + + RedisDumpMonitoring + OpsAdministration/dev/DataPipeline/RedisDumpMonitoring + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/ops/redis/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlert/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlert/config.xml index 946d75259..c4a2791b0 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlert/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlert/config.xml @@ -91,7 +91,7 @@ return """<b>This parameter is not used</b>"&quo - + */30 * * * * @@ -106,7 +106,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlertProvision/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlertProvision/config.xml index ffb576656..89b09c571 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlertProvision/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsAlertProvision/config.xml @@ -122,4 +122,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsKill/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsKill/config.xml index 293f2e6b9..e77ed5775 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsKill/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaJobsKill/config.xml @@ -123,7 +123,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaLogsBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaLogsBackup/config.xml index 879a1aa13..3dc7f04d8 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaLogsBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaLogsBackup/config.xml @@ -91,7 +91,7 @@ return """<b>This parameter is not used</b>"&quo - + 15 5 * * * diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaOrphanprocessAlert/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaOrphanprocessAlert/config.xml index a5700c418..8146d565f 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaOrphanprocessAlert/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaOrphanprocessAlert/config.xml @@ -47,7 +47,7 @@ - + 0 */2 * * * @@ -62,7 +62,7 @@ - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaThroughputAlert/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaThroughputAlert/config.xml index 48bd33ea8..d4d40ccfa 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaThroughputAlert/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/SamzaThroughputAlert/config.xml @@ -1,5 +1,5 @@ - + @@ -26,13 +26,13 @@ - + private_branch choice-parameter-2914866095851737 1 false - + false SamzaThroughputAlert + OpsAdministration/dev/DataPipeline/SamzaThroughputAlert ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-2914866097410763 1 false - + false SamzaThroughputAlert + OpsAdministration/dev/DataPipeline/SamzaThroughputAlert ET_FORMATTED_HTML @@ -97,13 +99,13 @@ return """<b>This parameter is not used</b>"&quo - + H/60 * * * * - - + + 2 @@ -112,7 +114,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false @@ -132,4 +134,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/config.xml index 6fe3ca0b6..ca27afff7 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/CassandraBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/CassandraBackup/config.xml index d54e0f9ac..234226a02 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/CassandraBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/CassandraBackup/config.xml @@ -97,7 +97,7 @@ return """<b>This parameter is not used</b>"&quo - + H 1 * * * @@ -112,7 +112,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchBackup/config.xml index 1dae910f7..d54244fc0 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/ElasticsearchBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + H 1 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Health/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Health/config.xml deleted file mode 100644 index 545f98faf..000000000 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Health/config.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - false - - - false - false - - - 0 - 0 - - false - project - false - - - - - - 2 - - - https://github.com/project-sunbird/sunbird-devops.git - - - - - ${branch_or_tag} - - - false - - - - pipelines/deploy/health-check/Jenkinsfile-lp - true - - - false - diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jBackup/config.xml index 498562efc..0648e1023 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/Neo4jBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + H 2 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/RedisBackup/config.xml b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/RedisBackup/config.xml index 5f3669a81..7bb0fedf9 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/RedisBackup/config.xml +++ b/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/KnowledgePlatform/jobs/RedisBackup/config.xml @@ -84,7 +84,7 @@ return """<b>This parameter is not used</b>"&quo - + H 2 * * * @@ -99,7 +99,7 @@ return """<b>This parameter is not used</b>"&quo - refs/tags/tag-for-backups + ${branch_or_tag} false diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/config.xml index 5be56a4e0..ca27afff7 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/JenkinsSlave/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/JenkinsSlave/config.xml new file mode 100644 index 000000000..3f3e9b8f3 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/JenkinsSlave/config.xml @@ -0,0 +1,70 @@ + + + + <font color=red size=2><b>This jobs is currently disabled as it's not being used.</b></font> + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + jenkins_host + <font color=dimgray size=2><b>Provide the IP of Jenkins host.</b></font> + + false + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/jenkins-slave/Jenkinsfile + false + + + true + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Keycloak/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Keycloak/config.xml index 69198f5fb..8b32f3a23 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Keycloak/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Keycloak/config.xml @@ -1,5 +1,5 @@ - + false @@ -13,19 +13,19 @@ - + false false - + private_branch choice-parameter-196487835595646 1 false - + false Keycloak + Provision/dev/Core/Keycloak ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-196487838166797 1 false - + false Keycloak + Provision/dev/Core/Keycloak ET_FORMATTED_HTML @@ -82,8 +84,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -112,4 +114,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/LogEsUpgrade6xx/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/LogEsUpgrade6xx/config.xml index 63db133d7..02a00d5d4 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/LogEsUpgrade6xx/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/LogEsUpgrade6xx/config.xml @@ -1,5 +1,5 @@ - + false @@ -13,44 +13,45 @@ - + false false - + private_branch choice-parameter-186487879604743 1 false - + false LogEsUpgrade6xx + Provision/loadtest/Core/LogEsUpgrade6xx ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-186487885338467 1 false - + false LogEsUpgrade6xx + Provision/loadtest/Core/LogEsUpgrade6xx ET_FORMATTED_HTML @@ -82,12 +84,12 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 - https://github.com/project-sunbird/sunbird-devops.git + https://github.com/kaali09/sunbird-devops.git @@ -112,4 +114,4 @@ return """<b>This parameter is not used</b>"&quo false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Registry/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Registry/config.xml new file mode 100644 index 000000000..7a4cd1955 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Registry/config.xml @@ -0,0 +1,115 @@ + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-196491343843987 + 1 + + false + + + + false + + + Registry + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196491346584243 + 1 + + false + + + + false + + + Registry + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/registry/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Secor/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Secor/config.xml new file mode 100644 index 000000000..102641d9f --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/Secor/config.xml @@ -0,0 +1,115 @@ + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-196492081645652 + 1 + + false + + + + false + + + Secor + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196492094618716 + 1 + + false + + + + false + + + Secor + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/secor/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisRestore/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/keycloak-test/config.xml similarity index 54% rename from deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisRestore/config.xml rename to deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/keycloak-test/config.xml index 03bd2126b..e335f972d 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/DataPipeline/jobs/RedisRestore/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/keycloak-test/config.xml @@ -4,22 +4,25 @@ false + + + -1 + -1 + -1 + 5 + + + false false - - redis_restore_file_name - redis backup file name which needs to be restored. - - false - private_branch - choice-parameter-4812402663506673 + choice-parameter-196487835595646 1 - redis-restore + keycloak-test ET_FORMATTED_HTML - false + true + + + branch_or_tag + + choice-parameter-196487838166797 + 1 + + false + + + + false + + + keycloak-test + + + ET_FORMATTED_HTML + true @@ -68,10 +96,18 @@ return """<b>This parameter is not used</b>"&quo false - + + + true + false + + 1 + false + + - pipelines/backup/redis-restore/Jenkinsfile - true + pipelines/provision/keycloak/Jenkinsfile + false false diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/user-keycloak/config.xml similarity index 83% rename from deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml rename to deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/user-keycloak/config.xml index 9b8aacb94..8e96c2c6c 100644 --- a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/user-keycloak/config.xml @@ -1,5 +1,5 @@ - + false @@ -13,7 +13,7 @@ - + false false @@ -22,23 +22,23 @@ private_branch - choice-parameter-186487879604743 + choice-parameter-196487835595646 1 false - + false - ApplicationElasticSearch + user-keycloak ET_FORMATTED_HTML @@ -47,10 +47,10 @@ return """<b>This parameter is not used</b>"&quo branch_or_tag - choice-parameter-186487885338467 + choice-parameter-196487838166797 1 false - + false - ApplicationElasticSearch + user-keycloak ET_FORMATTED_HTML @@ -82,8 +82,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -107,7 +107,7 @@ return """<b>This parameter is not used</b>"&quo - pipelines/deploy/application-elasticsearch/Jenkinsfile + pipelines/provision/keycloak/Jenkinsfile.user false diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/config.xml index 5be56a4e0..ca27afff7 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSpark/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSpark/config.xml index 24dcad973..3e78f2bae 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSpark/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/AnalyticsSpark/config.xml @@ -1,5 +1,5 @@ - + @@ -20,19 +20,19 @@ - + false false - + private_branch choice-parameter-196494527011530 1 false - + false AnalyticsSpark + Provision/dev/DataPipeline/AnalyticsSpark ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-196494532752488 1 false - + false AnalyticsSpark + Provision/dev/DataPipeline/AnalyticsSpark ET_FORMATTED_HTML @@ -80,6 +82,8 @@ return """<b>This parameter is not used</b>"&quo + 0 + 0 false project @@ -87,8 +91,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -117,4 +121,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Druid/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Druid/config.xml new file mode 100644 index 000000000..7d0c034d9 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Druid/config.xml @@ -0,0 +1,67 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/druid/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Kibana/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Kibana/config.xml new file mode 100644 index 000000000..cac92eae5 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Kibana/config.xml @@ -0,0 +1,120 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-196497749695581 + 1 + + false + + + + false + + + Kibana + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196497753237991 + 1 + + false + + + + false + + + Kibana + + + ET_FORMATTED_HTML + true + + + + + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/kibana/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/Health/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/MLWorkbench/config.xml similarity index 57% rename from deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/Health/config.xml rename to deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/MLWorkbench/config.xml index 1cd2aa21a..86592f868 100644 --- a/deploy/jenkins/jobs/OpsAdministration/jobs/dev/jobs/Core/jobs/Health/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/MLWorkbench/config.xml @@ -1,10 +1,19 @@ - + false - + + + -1 + -1 + -1 + 5 + + + + false false @@ -18,7 +27,7 @@ - + 2 @@ -33,10 +42,25 @@ false - + + + true + false + + 0 + false + + + true + false + + 0 + false + + - pipelines/deploy/health-check/Jenkinsfile - true + pipelines/provision/ml-workbench/Jenkinsfile + false false diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/NodeJs/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/NodeJs/config.xml new file mode 100644 index 000000000..9fa304888 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/NodeJs/config.xml @@ -0,0 +1,124 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-196492871381509 + 1 + + false + + + + false + + + NodeJs + Provision/dev/DataPipeline/NodeJs + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196492874659821 + 1 + + false + + + + false + + + NodeJs + Provision/dev/DataPipeline/NodeJs + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/analytics-api/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/RedisCluster/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/RedisCluster/config.xml index 3fb7ca18e..d65b18541 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/RedisCluster/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/RedisCluster/config.xml @@ -1,5 +1,5 @@ - + @@ -20,58 +20,60 @@ - + false false - + private_branch choice-parameter-196501654566824 1 - false + true - + false - Redis_Cluster + RedisCluster + Provision/dev/DataPipeline/RedisCluster ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-196501660936815 1 - false + true - + false - Redis_Cluster + RedisCluster + Provision/dev/DataPipeline/RedisCluster ET_FORMATTED_HTML @@ -92,12 +94,12 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 - https://github.com/project-sunbird/sunbird-data-pipeline.git + https://github.com/project-sunbird/sunbird-devops.git @@ -122,4 +124,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/TelemetrySearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/TelemetrySearch/config.xml new file mode 100644 index 000000000..d8fa81c34 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/TelemetrySearch/config.xml @@ -0,0 +1,122 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-196499327272611 + 1 + + false + + + + false + + + TelemetrySearch + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196499346601896 + 1 + + false + + + + false + + + TelemetrySearch + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/telemetry-elasticsearch/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml index f600d299c..f1baf8e9d 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/Yarn/config.xml @@ -1,5 +1,5 @@ - + @@ -20,58 +20,60 @@ - + false false - + private_branch choice-parameter-196500122471866 1 - false + true - + false Yarn + Provision/dev/DataPipeline/Yarn ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-196500128714203 1 - false + true - + - false + true Yarn + Provision/dev/DataPipeline/Yarn ET_FORMATTED_HTML @@ -80,6 +82,8 @@ return """<b>This parameter is not used</b>"&quo + 0 + 0 false project @@ -87,12 +91,12 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 - https://github.com/project-sunbird/sunbird-data-pipeline.git + https://github.com/project-sunbird/sunbird-data-pipeline @@ -117,4 +121,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/postgres-managed/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/postgres-managed/config.xml new file mode 100644 index 000000000..1a2bcd658 --- /dev/null +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/DataPipeline/jobs/postgres-managed/config.xml @@ -0,0 +1,123 @@ + + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + -1 + -1 + 5 + + + + false + false + + + + + private_branch + + choice-parameter-196498570819230 + 1 + + false + + + + false + + + postgres-managed + Provision/dev/DataPipeline/postgres-managed + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-196498577913500 + 1 + + false + + + + false + + + postgres-managed + Provision/dev/DataPipeline/postgres-managed + + + ET_FORMATTED_HTML + true + + + + + 0 + 0 + + false + project + false + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-data-pipeline.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/provision/postgres-managed/Jenkinsfile + false + + + false + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/config.xml index 68477b8ac..ca27afff7 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/config.xml @@ -1,11 +1,11 @@ - + - + - + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/Redis/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/Redis/config.xml index 758bcb5bb..8bb0df164 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/Redis/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/Redis/config.xml @@ -1,5 +1,5 @@ - + @@ -20,19 +20,19 @@ - + false false - + private_branch choice-parameter-196505638775314 1 false - + false Redis + Provision/dev/KnowledgePlatform/Redis ET_FORMATTED_HTML true - + branch_or_tag choice-parameter-196505642335779 1 false - + false Redis + Provision/dev/KnowledgePlatform/Redis ET_FORMATTED_HTML @@ -80,6 +82,8 @@ return """<b>This parameter is not used</b>"&quo + 0 + 0 false project @@ -90,8 +94,8 @@ return """<b>This parameter is not used</b>"&quo - - + + 2 @@ -120,4 +124,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/RedisCluster/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/RedisCluster/config.xml index ec9074fd8..b9b5b38d1 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/RedisCluster/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/KnowledgePlatform/jobs/RedisCluster/config.xml @@ -97,7 +97,7 @@ return """<b>This parameter is not used</b>"&quo 2 - https://github.com/project-sunbird/sunbird-learning-platform.git + https://github.com/project-sunbird/sunbird-devops.git @@ -122,4 +122,4 @@ return """<b>This parameter is not used</b>"&quo false - + \ No newline at end of file diff --git a/deploy/utilities/cassandra_row_counter.py b/deploy/utilities/cassandra_row_counter.py index 48a5f15c6..487689d76 100755 --- a/deploy/utilities/cassandra_row_counter.py +++ b/deploy/utilities/cassandra_row_counter.py @@ -16,7 +16,7 @@ try: for j in os.listdir(i): ks = i+'.'+str(j.split('-')[0]) - val = 'cqlsh -e "select count(*) from {};"'.format(ks) + val = 'cqlsh --request-timeout=36000 -e "select count(*) from {};"'.format(ks) try: out = check_output(["/bin/bash","-c",val], stderr=STDOUT) except Exception as e: diff --git a/exporters/Go/kafka-topic-exporter/Dockerfile b/exporters/Go/kafka-topic-exporter/Dockerfile new file mode 100644 index 000000000..c55c560f9 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/Dockerfile @@ -0,0 +1,5 @@ +FROM scratch +LABEL maintainer="Rajesh Rajendran" +ADD kafka-topic-exporter / +EXPOSE 8000 +CMD ["/kafka-topic-exporter"] diff --git a/exporters/Go/kafka-topic-exporter/README.md b/exporters/Go/kafka-topic-exporter/README.md new file mode 100644 index 000000000..d597d4a88 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/README.md @@ -0,0 +1,74 @@ +## Design + +This program is designed to be eventually consistent, metrics exporter. + +## What it does + +Scrape a kafka topic for metrics of specific json format, and expose one endpoint for prometheus + +## How to run + +`docker run -e kafaka_host=11.1.1.11:9092 -e kafka_topic=metrics.topic.dev rjshrjndrn/kafka-topic-exporter:v1` + +## Json format + +``` +{ + "system": "", + "subsystem": "", + "metricts": "< timestamp of the merics, which should be passed to prometheus. This should be in epoch milliseconds>", // This is an optional field + "metrics": [ + { + "id": "", // It can contain alphabets and '-' and '_'. Should should start with alphabet + "value": "< value of the metric>" // Should be of int or float64 + } + { + ... + ... + ... + } + ], + "dimensions": [ // Labels which will get injectd to each of the above metrics + { + "id": "< name of the label>", // It can contain alphabets and '-' and '_'. Should should start with alphabet + "value": < value of the label>" + } + { + ... + ... + ... + } + ] +} +``` + +Example: +``` +{ + "system": "samza", + "subsystem": "pipeline-metrics", + "metricts" : 1582521646464, + "metrics": [ + { + "id": "success-message-count", + "value": 1 + }, + { + "id": "skipped-message-count", + "value": 1 + }, + { + } + ], + "dimensions": [ + { + "id": "job-name", + "value": "test-job" + }, + { + "id": "partition", + "value": 0 + } + ] +} +``` diff --git a/exporters/Go/kafka-topic-exporter/build.sh b/exporters/Go/kafka-topic-exporter/build.sh new file mode 100755 index 000000000..f4fbcb0a0 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/build.sh @@ -0,0 +1,5 @@ +# vim: set ts=4 sw=4 tw=0 et : +#!/bin/bash + +CGO_ENABLED=0 go build -o kafka-topic-exporter main.go +docker build -f Dockerfile -t sunbird/kafka-topic-exporter:v1 . diff --git a/exporters/Go/kafka-topic-exporter/go.mod b/exporters/Go/kafka-topic-exporter/go.mod new file mode 100644 index 000000000..61e9945e8 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/go.mod @@ -0,0 +1,5 @@ +module kafka-topic-exporter/main + +go 1.13 + +require github.com/segmentio/kafka-go v0.3.7 diff --git a/exporters/Go/kafka-topic-exporter/go.sum b/exporters/Go/kafka-topic-exporter/go.sum new file mode 100644 index 000000000..2c1bf04a0 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/go.sum @@ -0,0 +1,15 @@ +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/segmentio/kafka-go v0.3.7 h1:UCFPJw6KoVkmrilA2LbWVuybJojHzj6gDDFdV7H7IBs= +github.com/segmentio/kafka-go v0.3.7/go.mod h1:8rEphJEczp+yDE/R5vwmaqZgF1wllrl4ioQcNKB8wVA= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/exporters/Go/kafka-topic-exporter/main.go b/exporters/Go/kafka-topic-exporter/main.go new file mode 100644 index 000000000..1f0ad3f28 --- /dev/null +++ b/exporters/Go/kafka-topic-exporter/main.go @@ -0,0 +1,272 @@ +// Author : Kaliraja Ramasami +// Author : Rajesh Rajendran +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "strings" + "sync" + "time" + + kafka "github.com/segmentio/kafka-go" + _ "github.com/segmentio/kafka-go/snappy" +) + +var kafkaReaders []*kafka.Reader + +// Channel to keep metrics till prometheus scrape that +var promMetricsChannel = make(chan metric) + +// Metrics structure +type Metrics struct { + System string `json:"system"` + SubSystem string `json:"subsystem"` + MetricTS json.Number `json:"metricTs"` + Metrics []struct { + ID string `json:"id"` + Value float64 `json:"value"` + } `json:"metrics"` + // Labels to be added for the metric + Lables []struct { + ID string `json:"id"` + // Even nimbers will be read as string + Value json.Number `json:"value"` + } `json:"dimensions"` +} + +// This is to get the last message served to prom endpoint. +type lastReadMessage struct { + mu sync.RWMutex + // map[partition]message + last map[int]kafka.Message + // slice pointer of kafkaReaders + clusterId int +} + +// Adding message +// Only the latest +func (lrm *lastReadMessage) Store(message kafka.Message, clusterId int) error { + lrm.mu.Lock() + defer lrm.mu.Unlock() + // Initializing map if nil + if lrm.last == nil { + lrm.last = make(map[int]kafka.Message) + } + if _, ok := lrm.last[message.Partition]; ok { + // Updating only if the offset is greater + if lrm.last[message.Partition].Offset < message.Offset { + lrm.last[message.Partition] = message + lrm.clusterId = clusterId + } + return fmt.Errorf("lower offset(%d) than the latest(%d)", message.Offset, lrm.last[message.Partition].Offset) + } else { + lrm.last[message.Partition] = message + lrm.clusterId = clusterId + } + return nil +} + +// Return the last message read +func (lrm *lastReadMessage) Get() (map[int]kafka.Message, int) { + lrm.mu.RLock() + defer lrm.mu.RUnlock() + return lrm.last, lrm.clusterId +} + +// Validating metrics name +// Input a list of string, and concatinate with _ after +// Removing all - in the provided names +// Convert mix cases to lowercases +func metricsNameValidator(names ...string) string { + retName := "" + for _, name := range names { + retName += strings.ReplaceAll(name, "-", "_") + "_" + } + return strings.ToLower(strings.TrimRight(retName, "_")) +} + +// Message format +type metric struct { + message string + id kafka.Message + clusterId int +} + +// This function will take the metrics input and create prometheus metrics +// output and send it to metrics channel +// So that http endpoint can serve the data +func (metrics *Metrics) pushMetrics(clusterId int, ctx context.Context, metricData *kafka.Message) (err error) { + label := fmt.Sprintf("system=%q,subsystem=%q,", metrics.System, metrics.SubSystem) + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Creating dictionary of labels + for _, labels := range metrics.Lables { + // Lables can't have '-' in it + label += fmt.Sprintf("%v=%q,", metricsNameValidator(labels.ID), labels.Value) + } + // Generating metrics + for _, m := range metrics.Metrics { + metricStruct := metric{} + // Adding optional timestamp + switch metrics.MetricTS { + case "": + metricStruct.message = fmt.Sprintf("%s{%s} %.2f", + metricsNameValidator(metrics.System, metrics.SubSystem, m.ID), + strings.TrimRight(label, ","), m.Value) + default: + metricStruct.message = fmt.Sprintf("%s{%s} %.1f %s", + metricsNameValidator(metrics.System, metrics.SubSystem, m.ID), + strings.TrimRight(label, ","), m.Value, metrics.MetricTS) + } + metricStruct.id = *metricData + metricStruct.clusterId = clusterId + promMetricsChannel <- metricStruct + } + return nil + } +} + +func metricsCreation(clusterId int, ctx context.Context, m kafka.Message) error { + metrics := Metrics{} + data := m.Value + // Creating metrics struct + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err := json.Unmarshal(data, &metrics); err != nil { + fmt.Printf("Unmarshal error: %q data: %q\n", err, string(data)) + return err + } + metrics.pushMetrics(clusterId, ctx, &m) + return nil + } +} + +func commitMessage(lastReadMessages *[]*lastReadMessage) error { + fmt.Println("number of clusters: ", len(*lastReadMessages)) + for _, lrm := range *lastReadMessages { + messages, clusterId := lrm.Get() + fmt.Println("cluster: ", clusterId) + for k, message := range messages { + if message.Offset <= 0 { + fmt.Println("Not committing anything", clusterId) + return errors.New("offset is not > 0 for cluster " + string(clusterId)) + } + fmt.Printf("Commiting message partition %d offset %d cluster %d\n", k, message.Offset, clusterId) + if err := kafkaReaders[clusterId].CommitMessages(context.Background(), message); err != nil { + return err + } + } + } + return nil +} + +func health(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte("{\"Healthy\": true}")) +} +func serve(w http.ResponseWriter, r *http.Request) { + fmt.Println("Serving Request") + ctx := r.Context() + var lastReadMessages []*lastReadMessage + // Reading topic + for k, v := range kafkaReaders { + lastReadMessage := lastReadMessage{} + lastReadMessages = append(lastReadMessages, &lastReadMessage) + go func(clusterId int, ctx context.Context, r *kafka.Reader) { + for { + // Only fetching the message, not commiting them + // It'll be commited once the transmission closes + m, err := r.FetchMessage(ctx) + if err != nil { + fmt.Printf("err reading message: %v\n", err) + break + } + fmt.Printf("topic: %q partition: %v offset: %v\n", m.Topic, m.Partition, m.Offset) + go func(clusterId int, ctx context.Context, m kafka.Message) { + if err := metricsCreation(clusterId, ctx, m); err != nil { + fmt.Printf("errored out metrics creation; err: %s\n", err) + return + } + }(clusterId, ctx, m) + } + }(k, ctx, v) + } + for { + select { + case message := <-promMetricsChannel: + fmt.Fprintf(w, "%s\n", message.message) + lastReadMessages[message.clusterId].Store(message.id, message.clusterId) + case <-ctx.Done(): + commitMessage(&lastReadMessages) + return + case <-time.After(1 * time.Second): + commitMessage(&lastReadMessages) + return + } + } +} + +func main() { + // Getting kafka_ip and topic + kafkaHosts := strings.Split(os.Getenv("kafka_host"), ";") + kafkaTopic := strings.Split(os.Getenv("kafka_topic"), ";") + kafkaConsumerGroupName := os.Getenv("kafka_consumer_group_name") + if kafkaConsumerGroupName == "" { + kafkaConsumerGroupName = "prometheus-metrics-consumer" + } + if kafkaTopic[0] == "" || kafkaHosts[0] == "" { + log.Fatalf(`"kafka_topic or kafka_host environment variables not set." +For example, + # export kafka_host=cluster1ip1:9092,cluster1ip2:9092;cluster2ip1:9092,cluster2ip2:9092 + # export kafka_topic=cluster1topic;cluster2topic + # ',' seperated multiple kafka nodes in the cluster and + # ';' seperated multiple kafka clusters + export kafka_host=10.0.0.9:9092,10.0.0.10:9092;20.0.0.9:9092,20.0.0.10:9092 + export kafka_topic=sunbird.metrics.topic;myapp.metrics.topic`) + } + fmt.Printf("kafka_host: %s\nkafka_topic: %s\nkafka_consumer_group_name: %s\n", kafkaHosts, kafkaTopic, kafkaConsumerGroupName) + // Checking kafka topics are given for all kafka clusters + if len(kafkaHosts) != len(kafkaTopic) { + log.Fatal("You should give same number of kafka_topics as kafka_clusters") + } + // Checking kafka port and ip are accessible + fmt.Println("Checking connection to kafka") + for _, hosts := range kafkaHosts { + for _, host := range strings.Split(hosts, ",") { + conn, err := net.DialTimeout("tcp", host, 10*time.Second) + if err != nil { + log.Fatalf("Connection error: %s", err) + } + fmt.Println("connection succeeded", host) + conn.Close() + } + } + fmt.Println("kafka is accessible") + // Initializing kafka + for k, v := range kafkaHosts { + kafkaReaders = append(kafkaReaders, kafka.NewReader(kafka.ReaderConfig{ + Brokers: strings.Split(v, ","), + GroupID: kafkaConsumerGroupName, // Consumer group ID + Topic: kafkaTopic[k], + MinBytes: 1e3, // 1KB + MaxBytes: 10e6, // 10MB + MaxWait: 200 * time.Millisecond, + RebalanceTimeout: time.Second * 5, + })) + defer kafkaReaders[k].Close() + } + http.HandleFunc("/metrics", serve) + http.HandleFunc("/health", health) + log.Fatal(http.ListenAndServe(":8000", nil)) +} diff --git a/images/kafka-topic-exporter/Dockerfile b/images/kafka-topic-exporter/Dockerfile new file mode 100644 index 000000000..ca21bff48 --- /dev/null +++ b/images/kafka-topic-exporter/Dockerfile @@ -0,0 +1,11 @@ +FROM golang:1.13.10 AS builder +WORKDIR /work +COPY main.go /work +RUN go get github.com/segmentio/kafka-go ; go get github.com/segmentio/kafka-go/snappy +RUN CGO_ENABLED=0 go build -o kafka-topic-exporter main.go + +FROM scratch +LABEL maintainer="Rajesh Rajendran" +COPY --from=builder /work/kafka-topic-exporter / +EXPOSE 8000 +CMD ["/kafka-topic-exporter"] diff --git a/images/kafka-topic-exporter/Jenkinsfile b/images/kafka-topic-exporter/Jenkinsfile new file mode 100644 index 000000000..7f23198a3 --- /dev/null +++ b/images/kafka-topic-exporter/Jenkinsfile @@ -0,0 +1,50 @@ +node('build-slave') { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + ansiColor('xterm') { + stage('Checkout') { + if (!env.hub_org) { + println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL) + error 'Please resolve the errors and rerun..' + } else + println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL) + } + sh'[[ -d .git ]] && git clean -ffxd' + checkout scm + if (params.github_release_tag == "") { + checkout scm + commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() + branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim() + build_tag = branch_name + "_" + commit_hash + "_" + env.BUILD_NUMBER + println(ANSI_BOLD + ANSI_YELLOW + "Tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL) + } else { + def scmVars = checkout scm + checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]] + build_tag = params.github_release_tag + "_" + env.BUILD_NUMBER + println(ANSI_BOLD + ANSI_YELLOW + "Tag specified, building from tag: " + params.github_release_tag + ANSI_NORMAL) + } + echo "build_tag: " + build_tag + + stage('Build') { + env.NODE_ENV = "build" + print "Environment will be : ${env.NODE_ENV}" + sh("cd ./images/kafka-topic-exporter && ./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}") + } + stage('ArchiveArtifacts') { + archiveArtifacts "images/kafka-topic-exporter/metadata.json" + currentBuild.description = "${build_tag}" + } + } + + } + catch (err) { + currentBuild.result = "FAILURE" + throw err + } + +} diff --git a/images/kafka-topic-exporter/build.sh b/images/kafka-topic-exporter/build.sh new file mode 100755 index 000000000..f3d6c24e7 --- /dev/null +++ b/images/kafka-topic-exporter/build.sh @@ -0,0 +1,6 @@ +# vim: set ts=4 sw=4 tw=0 et : +#!/bin/bash + +# Have to copy this file because In docker container we can't pass directories other than PWD +cp ../../exporters/Go/kafka-topic-exporter/{main.go,go.mod,go.sum} . +docker build -f Dockerfile -t sunbird/kafka-topic-exporter:v3 . diff --git a/images/kafka-topic-exporter/metadata.sh b/images/kafka-topic-exporter/metadata.sh new file mode 100755 index 000000000..f22c55764 --- /dev/null +++ b/images/kafka-topic-exporter/metadata.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# return version +echo '{"name":"kafka-topic-exporter","version":"1.5.0","org":"sunbird","hubuser":"purplesunbird"}' diff --git a/images/kong/Dockerfile b/images/kong/Dockerfile index 4fd741f59..b2b66e0ac 100644 --- a/images/kong/Dockerfile +++ b/images/kong/Dockerfile @@ -1,35 +1,22 @@ -#FROM kong:0.9.9 +FROM kong:0.14.1-centos -# ensure Kong logs go to the log pipe from our entrypoint and so to docker logging -#RUN mkdir -p /usr/local/kong/logs \ -# && ln -sf /dev/stdout /usr/local/kong/logs/serf.log \ -# && ln -sf /dev/stderr /usr/local/kong/logs/error.log \ -# && ln -sf /dev/stdout /usr/local/kong/logs/access.log - - -#========================================================================= - - -FROM centos:7 - -ENV KONG_VERSION 0.9.9 - -RUN yum install -y wget https://github.com/Mashape/kong/releases/download/$KONG_VERSION/kong-$KONG_VERSION.el7.noarch.rpm && \ - yum clean all - -RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64 && \ - chmod +x /usr/local/bin/dumb-init - -RUN yum install -y net-tools +ENV KONG_VERSION 0.14.1 COPY docker-entrypoint.sh /docker-entrypoint.sh ENTRYPOINT ["/docker-entrypoint.sh"] +COPY plugins/$KONG_VERSION/jwt/* /usr/local/share/lua/5.1/kong/plugins/jwt/ +#COPY plugins/$KONG_VERSION/prometheus/* /usr/local/share/lua/5.1/kong/plugins/prometheus/ +COPY plugins/$KONG_VERSION/rate-limiting/policies/* /usr/local/share/lua/5.1/kong/plugins/rate-limiting/policies/ +COPY templates/$KONG_VERSION/nginx_kong.lua /usr/local/share/lua/5.1/kong/templates/ +COPY templates/$KONG_VERSION/kong_defaults.lua /usr/local/share/lua/5.1/kong/templates/ + # ensure Kong logs go to the log pipe from our entrypoint and so to docker logging RUN mkdir -p /usr/local/kong/logs \ && ln -sf /tmp/logpipe /usr/local/kong/logs/access.log \ - && ln -sf /tmp/logpipe /usr/local/kong/logs/serf.log \ - && ln -sf /tmp/logpipe /usr/local/kong/logs/error.log + && ln -sf /tmp/logpipe /usr/local/kong/logs/error.log \ + && ln -sf /tmp/logpipe /usr/local/kong/logs/admin_access.log EXPOSE 8000 8443 8001 7946 CMD ["kong", "start"] + diff --git a/images/kong/Jenkinsfile.build b/images/kong/Jenkinsfile.build index 0a4733291..c576900a3 100644 --- a/images/kong/Jenkinsfile.build +++ b/images/kong/Jenkinsfile.build @@ -36,8 +36,7 @@ node('build-slave') { echo "build_tag: " + build_tag stage('Build') { - // This image version is not updating always - def build_tag = '2.0.0' + build_tag = "0.14.1" sh("cd images/kong/ && ./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}") } diff --git a/images/kong/docker-entrypoint.sh b/images/kong/docker-entrypoint.sh index 773f0a22d..09c0c1140 100755 --- a/images/kong/docker-entrypoint.sh +++ b/images/kong/docker-entrypoint.sh @@ -1,23 +1,19 @@ -#!/usr/local/bin/dumb-init /bin/bash +#!/bin/sh set -e - -# Make a pipe for the logs so we can ensure Kong logs get directed to docker logging -# see https://github.com/docker/docker/issues/6880 -# also, https://github.com/docker/docker/issues/31106, https://github.com/docker/docker/issues/31243 -# https://github.com/docker/docker/pull/16468, https://github.com/behance/docker-nginx/pull/51 rm -f /tmp/logpipe mkfifo -m 666 /tmp/logpipe # This child process will still receive signals as per https://github.com/Yelp/dumb-init#session-behavior cat <> /tmp/logpipe 1>&1 & -# NOTE, to configure the `File Log` plugin to route logs to Docker logging, direct `config.path` at `/tmp/logpipe` - # Disabling nginx daemon mode export KONG_NGINX_DAEMON="off" -# Ensure kong listens on correct ip address https://github.com/Mashape/docker-kong/issues/93 -IP_ADDR=`ifconfig eth0 | awk '$1 == "inet" {gsub(/\/.*$/, "", $2); print $2}'` -export KONG_CLUSTER_LISTEN="$IP_ADDR:7946" -echo "KONG_CLUSTER_LISTEN: $KONG_CLUSTER_LISTEN" +# Setting default prefix (override any existing variable) +export KONG_PREFIX="/usr/local/kong" + +# Prepare Kong prefix +if [ "$1" = "/usr/local/openresty/nginx/sbin/nginx" ]; then + kong prepare -p "/usr/local/kong" +fi exec "$@" diff --git a/images/kong/plugins/0.12.3/jwt/handler.lua b/images/kong/plugins/0.12.3/jwt/handler.lua new file mode 100644 index 000000000..81fbdade9 --- /dev/null +++ b/images/kong/plugins/0.12.3/jwt/handler.lua @@ -0,0 +1,231 @@ +local singletons = require "kong.singletons" +local BasePlugin = require "kong.plugins.base_plugin" +local responses = require "kong.tools.responses" +local constants = require "kong.constants" +local jwt_decoder = require "kong.plugins.jwt.jwt_parser" + +local ipairs = ipairs +local string_format = string.format +local ngx_re_gmatch = ngx.re.gmatch +local ngx_set_header = ngx.req.set_header +local get_method = ngx.req.get_method + +local JwtHandler = BasePlugin:extend() + +JwtHandler.PRIORITY = 1005 +JwtHandler.VERSION = "0.1.0" + +--- Retrieve a JWT in a request. +-- Checks for the JWT in URI parameters, then in cookies, and finally +-- in the `Authorization` header. +-- @param request ngx request object +-- @param conf Plugin configuration +-- @return token JWT token contained in request (can be a table) or nil +-- @return err +local function retrieve_token(request, conf) + local uri_parameters = request.get_uri_args() + + for _, v in ipairs(conf.uri_param_names) do + if uri_parameters[v] then + return uri_parameters[v] + end + end + + local ngx_var = ngx.var + for _, v in ipairs(conf.cookie_names) do + local jwt_cookie = ngx_var["cookie_" .. v] + if jwt_cookie and jwt_cookie ~= "" then + return jwt_cookie + end + end + + local authorization_header = request.get_headers()["authorization"] + if authorization_header then + local iterator, iter_err = ngx_re_gmatch(authorization_header, "\\s*[Bb]earer\\s+(.+)") + if not iterator then + return nil, iter_err + end + + local m, err = iterator() + if err then + return nil, err + end + + if m and #m > 0 then + return m[1] + end + end +end + +function JwtHandler:new() + JwtHandler.super.new(self, "jwt") +end + +local function load_credential(jwt_secret_key) + local rows, err = singletons.dao.jwt_secrets:find_all {key = jwt_secret_key} + if err then + return nil, err + end + return rows[1] +end + +local function load_consumer(consumer_id, anonymous) + local result, err = singletons.dao.consumers:find { id = consumer_id } + if not result then + if anonymous and not err then + err = 'anonymous consumer "' .. consumer_id .. '" not found' + end + return nil, err + end + return result +end + +local function set_consumer(consumer, jwt_secret) + ngx_set_header(constants.HEADERS.CONSUMER_ID, consumer.id) + ngx_set_header(constants.HEADERS.CONSUMER_CUSTOM_ID, consumer.custom_id) + ngx_set_header(constants.HEADERS.CONSUMER_USERNAME, consumer.username) + ngx.ctx.authenticated_consumer = consumer + + if jwt_secret then + ngx.ctx.authenticated_credential = jwt_secret + ngx_set_header(constants.HEADERS.ANONYMOUS, nil) -- in case of auth plugins concatenation + else + ngx_set_header(constants.HEADERS.ANONYMOUS, true) + end + +end + +local function do_authentication(conf) + local token, err = retrieve_token(ngx.req, conf) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + local ttype = type(token) + if ttype ~= "string" then + if ttype == "nil" then + return false, {status = 401} + elseif ttype == "table" then + return false, {status = 401, message = "Multiple tokens provided"} + else + return false, {status = 401, message = "Unrecognizable token"} + end + end + -- Decode token to find out who the consumer is + local jwt, err = jwt_decoder:new(token) + if err then + -- Don't sent Bad Token for null / empty Bearer tokens + return false, {status = 401} + end + + local claims = jwt.claims + + local jwt_secret_key = claims[conf.key_claim_name] + if not jwt_secret_key then + return false, {status = 401, message = "No mandatory '" .. conf.key_claim_name .. "' in claims"} + end + + -- If the bearer key contains a kid, change the secret key + if jwt.header.kid then + jwt_secret_key = jwt.header.kid + end + + -- Retrieve the secret + local jwt_secret_cache_key = singletons.dao.jwt_secrets:cache_key(jwt_secret_key) + local jwt_secret, err = singletons.cache:get(jwt_secret_cache_key, nil, + load_credential, jwt_secret_key) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + if not jwt_secret then + return false, {status = 403, message = "No credentials found for given '" .. conf.key_claim_name .. "'"} + end + + local algorithm = jwt_secret.algorithm or "HS256" + + -- Verify "alg" + if jwt.header.alg ~= algorithm then + return false, {status = 403, message = "Invalid algorithm"} + end + + local jwt_secret_value = algorithm == "HS256" and jwt_secret.secret or jwt_secret.rsa_public_key + if conf.secret_is_base64 then + jwt_secret_value = jwt:b64_decode(jwt_secret_value) + end + + if not jwt_secret_value then + return false, {status = 403, message = "Invalid key/secret"} + end + + -- Now verify the JWT signature + if not jwt:verify_signature(jwt_secret_value) then + return false, {status = 403, message = "Invalid signature"} + end + + -- Verify the JWT registered claims + local ok_claims, errors = jwt:verify_registered_claims(conf.claims_to_verify) + if not ok_claims then + return false, {status = 401, message = errors} + end + + -- Retrieve the consumer + local consumer_cache_key = singletons.dao.consumers:cache_key(jwt_secret.consumer_id) + local consumer, err = singletons.cache:get(consumer_cache_key, nil, + load_consumer, + jwt_secret.consumer_id, true) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + -- However this should not happen + if not consumer then + return false, {status = 403, message = string_format("Could not find consumer for '%s=%s'", conf.key_claim_name, jwt_secret_key)} + end + +-- Restore the orignal key and update the id post operation + if jwt.header.kid then + jwt_secret.id = jwt.claims["iss"] + jwt_secret.key = jwt.claims["iss"] + end + + set_consumer(consumer, jwt_secret) + + return true +end + + +function JwtHandler:access(conf) + JwtHandler.super.access(self) + + -- check if preflight request and whether it should be authenticated + if not conf.run_on_preflight and get_method() == "OPTIONS" then + return + end + + if ngx.ctx.authenticated_credential and conf.anonymous ~= "" then + -- we're already authenticated, and we're configured for using anonymous, + -- hence we're in a logical OR between auth methods and we're already done. + return + end + + local ok, err = do_authentication(conf) + if not ok then + if conf.anonymous ~= "" then + -- get anonymous user + local consumer_cache_key = singletons.dao.consumers:cache_key(conf.anonymous) + local consumer, err = singletons.cache:get(consumer_cache_key, nil, + load_consumer, + conf.anonymous, true) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + set_consumer(consumer, nil) + else + return responses.send(err.status, err.message) + end + end +end + + +return JwtHandler diff --git a/images/kong/plugins/0.14.1/jwt/handler.lua b/images/kong/plugins/0.14.1/jwt/handler.lua new file mode 100644 index 000000000..390e0da1b --- /dev/null +++ b/images/kong/plugins/0.14.1/jwt/handler.lua @@ -0,0 +1,242 @@ +local singletons = require "kong.singletons" +local BasePlugin = require "kong.plugins.base_plugin" +local responses = require "kong.tools.responses" +local constants = require "kong.constants" +local jwt_decoder = require "kong.plugins.jwt.jwt_parser" + +local ipairs = ipairs +local string_format = string.format +local ngx_re_gmatch = ngx.re.gmatch +local ngx_set_header = ngx.req.set_header +local get_method = ngx.req.get_method + +local JwtHandler = BasePlugin:extend() + +JwtHandler.PRIORITY = 1005 +JwtHandler.VERSION = "0.1.0" + +--- Retrieve a JWT in a request. +-- Checks for the JWT in URI parameters, then in cookies, and finally +-- in the `Authorization` header. +-- @param request ngx request object +-- @param conf Plugin configuration +-- @return token JWT token contained in request (can be a table) or nil +-- @return err +local function retrieve_token(request, conf) + local uri_parameters = request.get_uri_args() + + for _, v in ipairs(conf.uri_param_names) do + if uri_parameters[v] then + return uri_parameters[v] + end + end + + local ngx_var = ngx.var + for _, v in ipairs(conf.cookie_names) do + local jwt_cookie = ngx_var["cookie_" .. v] + if jwt_cookie and jwt_cookie ~= "" then + return jwt_cookie + end + end + + local authorization_header = request.get_headers()["authorization"] + if authorization_header then + local iterator, iter_err = ngx_re_gmatch(authorization_header, "\\s*[Bb]earer\\s+(.+)") + if not iterator then + return nil, iter_err + end + + local m, err = iterator() + if err then + return nil, err + end + + if m and #m > 0 then + return m[1] + end + end +end + +function JwtHandler:new() + JwtHandler.super.new(self, "jwt") +end + +local function load_credential(jwt_secret_key) + local rows, err = singletons.dao.jwt_secrets:find_all {key = jwt_secret_key} + if err then + return nil, err + end + return rows[1] +end + +local function load_consumer(consumer_id, anonymous) + local result, err = singletons.db.consumers:select { id = consumer_id } + if not result then + if anonymous and not err then + err = 'anonymous consumer "' .. consumer_id .. '" not found' + end + return nil, err + end + return result +end + +local function set_consumer(consumer, jwt_secret, token) + ngx_set_header(constants.HEADERS.CONSUMER_ID, consumer.id) + ngx_set_header(constants.HEADERS.CONSUMER_CUSTOM_ID, consumer.custom_id) + ngx_set_header(constants.HEADERS.CONSUMER_USERNAME, consumer.username) + ngx.ctx.authenticated_consumer = consumer + if jwt_secret then + ngx.ctx.authenticated_credential = jwt_secret + ngx.ctx.authenticated_jwt_token = token + ngx_set_header(constants.HEADERS.ANONYMOUS, nil) -- in case of auth plugins concatenation + else + ngx_set_header(constants.HEADERS.ANONYMOUS, true) + end + +end + +local function do_authentication(conf) + local token, err = retrieve_token(ngx.req, conf) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + local ttype = type(token) + if ttype ~= "string" then + if ttype == "nil" then + return false, {status = 401} + elseif ttype == "table" then + return false, {status = 401, message = "Multiple tokens provided"} + else + return false, {status = 401, message = "Unrecognizable token"} + end + end + + -- Decode token to find out who the consumer is + local jwt, err = jwt_decoder:new(token) + if err then + -- Don't sent Bad Token for null / empty Bearer tokens + return false, {status = 401} + end + + local claims = jwt.claims + local header = jwt.header + + local jwt_secret_key = claims[conf.key_claim_name] or header[conf.key_claim_name] + if not jwt_secret_key then + return false, {status = 401, message = "No mandatory '" .. conf.key_claim_name .. "' in claims"} + end + + -- If the bearer key contains a kid, change the secret key + if jwt.header.kid then + jwt_secret_key = jwt.header.kid + end + + -- Retrieve the secret + local jwt_secret_cache_key = singletons.dao.jwt_secrets:cache_key(jwt_secret_key) + local jwt_secret, err = singletons.cache:get(jwt_secret_cache_key, nil, + load_credential, jwt_secret_key) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + if not jwt_secret then + return false, {status = 403, message = "No credentials found for given '" .. conf.key_claim_name .. "'"} + end + + local algorithm = jwt_secret.algorithm or "HS256" + + -- Verify "alg" + if jwt.header.alg ~= algorithm then + return false, {status = 403, message = "Invalid algorithm"} + end + + local jwt_secret_value = algorithm ~= nil and algorithm:sub(1, 2) == "HS" and + jwt_secret.secret or jwt_secret.rsa_public_key + if conf.secret_is_base64 then + jwt_secret_value = jwt:b64_decode(jwt_secret_value) + end + + if not jwt_secret_value then + return false, {status = 403, message = "Invalid key/secret"} + end + + -- Now verify the JWT signature + if not jwt:verify_signature(jwt_secret_value) then + return false, {status = 403, message = "Invalid signature"} + end + + -- Verify the JWT registered claims + local ok_claims, errors = jwt:verify_registered_claims(conf.claims_to_verify) + if not ok_claims then + return false, {status = 401, message = errors} + end + + -- Verify the JWT registered claims + if conf.maximum_expiration ~= nil and conf.maximum_expiration > 0 then + local ok, errors = jwt:check_maximum_expiration(conf.maximum_expiration) + if not ok then + return false, {status = 403, message = errors} + end + end + + -- Retrieve the consumer + local consumer_cache_key = singletons.db.consumers:cache_key(jwt_secret.consumer_id) + local consumer, err = singletons.cache:get(consumer_cache_key, nil, + load_consumer, + jwt_secret.consumer_id, true) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + + -- However this should not happen + if not consumer then + return false, {status = 403, message = string_format("Could not find consumer for '%s=%s'", conf.key_claim_name, jwt_secret_key)} + end + + -- Restore the orignal key and update the id post operation + if jwt.header.kid then + jwt_secret.id = jwt.claims["iss"] + jwt_secret.key = jwt.claims["iss"] + end + + set_consumer(consumer, jwt_secret, token) + + return true +end + + +function JwtHandler:access(conf) + JwtHandler.super.access(self) + + -- check if preflight request and whether it should be authenticated + if not conf.run_on_preflight and get_method() == "OPTIONS" then + return + end + + if ngx.ctx.authenticated_credential and conf.anonymous ~= "" then + -- we're already authenticated, and we're configured for using anonymous, + -- hence we're in a logical OR between auth methods and we're already done. + return + end + + local ok, err = do_authentication(conf) + if not ok then + if conf.anonymous ~= "" then + -- get anonymous user + local consumer_cache_key = singletons.db.consumers:cache_key(conf.anonymous) + local consumer, err = singletons.cache:get(consumer_cache_key, nil, + load_consumer, + conf.anonymous, true) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + set_consumer(consumer, nil, nil) + else + return responses.send(err.status, err.message) + end + end +end + + +return JwtHandler diff --git a/images/kong/plugins/0.14.1/prometheus/api.lua b/images/kong/plugins/0.14.1/prometheus/api.lua new file mode 100644 index 000000000..5ba32904e --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/api.lua @@ -0,0 +1,9 @@ +local prometheus = require "kong.plugins.prometheus.exporter" + +return { + ["/metrics"] = { + GET = function() + prometheus.collect() + end, + }, +} diff --git a/images/kong/plugins/0.14.1/prometheus/exporter.lua b/images/kong/plugins/0.14.1/prometheus/exporter.lua new file mode 100644 index 000000000..07f5b91c4 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/exporter.lua @@ -0,0 +1,256 @@ +local kong = kong +local ngx = ngx +local find = string.find +local lower = string.lower +local concat = table.concat +local select = select +local balancer = require("kong.runloop.balancer") + +local DEFAULT_BUCKETS = { 1, 2, 5, 7, 10, 15, 20, 25, 30, 40, 50, 60, 70, + 80, 90, 100, 200, 300, 400, 500, 1000, + 2000, 5000, 10000, 30000, 60000 } +local metrics = {} +-- prometheus.lua instance +local prometheus + +-- use the same counter library shipped with Kong +package.loaded['prometheus_resty_counter'] = require("resty.counter") + + +local function init() + local shm = "prometheus_metrics" + if not ngx.shared.prometheus_metrics then + kong.log.err("prometheus: ngx shared dict 'prometheus_metrics' not found") + return + end + + prometheus = require("kong.plugins.prometheus.prometheus").init(shm, "kong_") + + -- global metrics + metrics.connections = prometheus:gauge("nginx_http_current_connections", + "Number of HTTP connections", + {"state"}) + metrics.db_reachable = prometheus:gauge("datastore_reachable", + "Datastore reachable from Kong, " .. + "0 is unreachable") + metrics.upstream_target_health = prometheus:gauge("upstream_target_health", + "Health status of targets of upstream. " .. + "States = healthchecks_off|healthy|unhealthy|dns_error, " .. + "value is 1 when state is populated.", + {"upstream", "target", "address", "state"}) + + local memory_stats = {} + memory_stats.worker_vms = prometheus:gauge("memory_workers_lua_vms_bytes", + "Allocated bytes in worker Lua VM", + {"pid"}) + memory_stats.shms = prometheus:gauge("memory_lua_shared_dict_bytes", + "Allocated slabs in bytes in a shared_dict", + {"shared_dict"}) + memory_stats.shm_capacity = prometheus:gauge("memory_lua_shared_dict_total_bytes", + "Total capacity in bytes of a shared_dict", + {"shared_dict"}) + + local res = kong.node.get_memory_stats() + for shm_name, value in pairs(res.lua_shared_dicts) do + memory_stats.shm_capacity:set(value.capacity, {shm_name}) + end + + metrics.memory_stats = memory_stats + + -- per service/route + metrics.status = prometheus:counter("http_status", + "HTTP status codes per service/route in Kong", + {"service", "route", "code"}) + metrics.latency = prometheus:histogram("latency", + "Latency added by Kong, total " .. + "request time and upstream latency " .. + "for each service/route in Kong", + {"service", "route", "type"}, + DEFAULT_BUCKETS) -- TODO make this configurable + metrics.bandwidth = prometheus:counter("bandwidth", + "Total bandwidth in bytes " .. + "consumed per service/route in Kong", + {"service", "route", "type"}) +end + +local function init_worker() + prometheus:init_worker() +end + + +-- Since in the prometheus library we create a new table for each diverged label +-- so putting the "more dynamic" label at the end will save us some memory +local labels_table = {0, 0, 0} +local upstream_target_addr_health_table = { + { value = 0, labels = { 0, 0, 0, "healthchecks_off" } }, + { value = 0, labels = { 0, 0, 0, "healthy" } }, + { value = 0, labels = { 0, 0, 0, "unhealthy" } }, + { value = 0, labels = { 0, 0, 0, "dns_error" } }, +} + +local function set_healthiness_metrics(table, upstream, target, address, status, metrics_bucket) + for i = 1, #table do + table[i]['labels'][1] = upstream + table[i]['labels'][2] = target + table[i]['labels'][3] = address + table[i]['value'] = (status == table[i]['labels'][4]) and 1 or 0 + metrics_bucket:set(table[i]['value'], table[i]['labels']) + end +end + +local function log(message) + if not metrics then + kong.log.err("prometheus: can not log metrics because of an initialization " + .. "error, please make sure that you've declared " + .. "'prometheus_metrics' shared dict in your nginx template") + return + end + + local service_name + if message and message.service then + service_name = message.service.name or message.service.host + else + -- do not record any stats if the service is not present + return + end + + local route_name + if message and message.route then + route_name = message.route.name or message.route.id + end + + labels_table[1] = service_name + labels_table[2] = route_name + labels_table[3] = message.response.status + metrics.status:inc(1, labels_table) + + local request_size = tonumber(message.request.size) + if request_size and request_size > 0 then + labels_table[3] = "ingress" + metrics.bandwidth:inc(request_size, labels_table) + end + + local response_size = tonumber(message.response.size) + if response_size and response_size > 0 then + labels_table[3] = "egress" + metrics.bandwidth:inc(response_size, labels_table) + end + + local request_latency = message.latencies.request + if request_latency and request_latency >= 0 then + labels_table[3] = "request" + metrics.latency:observe(request_latency, labels_table) + end + + local upstream_latency = message.latencies.proxy + if upstream_latency ~= nil and upstream_latency >= 0 then + labels_table[3] = "upstream" + metrics.latency:observe(upstream_latency, labels_table) + end + + local kong_proxy_latency = message.latencies.kong + if kong_proxy_latency ~= nil and kong_proxy_latency >= 0 then + labels_table[3] = "kong" + metrics.latency:observe(kong_proxy_latency, labels_table) + end +end + + +local function collect() + if not prometheus or not metrics then + kong.log.err("prometheus: plugin is not initialized, please make sure ", + " 'prometheus_metrics' shared dict is present in nginx template") + return kong.response.exit(500, { message = "An unexpected error occurred" }) + end + + local r = ngx.location.capture "/nginx_status" + + if r.status ~= 200 then + kong.log.warn("prometheus: failed to retrieve /nginx_status ", + "while processing /metrics endpoint") + + else + local accepted, handled, total = select(3, find(r.body, + "accepts handled requests\n (%d*) (%d*) (%d*)")) + metrics.connections:set(accepted, { "accepted" }) + metrics.connections:set(handled, { "handled" }) + metrics.connections:set(total, { "total" }) + end + + metrics.connections:set(ngx.var.connections_active, { "active" }) + metrics.connections:set(ngx.var.connections_reading, { "reading" }) + metrics.connections:set(ngx.var.connections_writing, { "writing" }) + metrics.connections:set(ngx.var.connections_waiting, { "waiting" }) + + -- db reachable? + local ok, err = kong.db.connector:connect() + if ok then + metrics.db_reachable:set(1) + + else + metrics.db_reachable:set(0) + kong.log.err("prometheus: failed to reach database while processing", + "/metrics endpoint: ", err) + end + + -- erase all target/upstream metrics, prevent exposing old metrics + metrics.upstream_target_health:reset() + + -- upstream targets accessible? + local upstreams_dict = balancer.get_all_upstreams() + for key, upstream_id in pairs(upstreams_dict) do + local _, upstream_name = key:match("^([^:]*):(.-)$") + upstream_name = upstream_name and upstream_name or key + -- based on logic from kong.db.dao.targets + local health_info + health_info, err = balancer.get_upstream_health(upstream_id) + if err then + kong.log.err("failed getting upstream health: ", err) + end + + if health_info then + for target_name, target_info in pairs(health_info) do + if target_info ~= nil and target_info.addresses ~= nil and + #target_info.addresses > 0 then + -- healthchecks_off|healthy|unhealthy + for _, address in ipairs(target_info.addresses) do + local address_label = concat({address.ip, ':', address.port}) + local status = lower(address.health) + set_healthiness_metrics(upstream_target_addr_health_table, upstream_name, target_name, address_label, status, metrics.upstream_target_health) + end + else + -- dns_error + set_healthiness_metrics(upstream_target_addr_health_table, upstream_name, target_name, '', 'dns_error', metrics.upstream_target_health) + end + end + end + end + + -- memory stats + local res = kong.node.get_memory_stats() + for shm_name, value in pairs(res.lua_shared_dicts) do + metrics.memory_stats.shms:set(value.allocated_slabs, {shm_name}) + end + for i = 1, #res.workers_lua_vms do + metrics.memory_stats.worker_vms:set(res.workers_lua_vms[i].http_allocated_gc, + {res.workers_lua_vms[i].pid}) + end + + prometheus:collect() +end + +local function get_prometheus() + if not prometheus then + kong.log.err("prometheus: plugin is not initialized, please make sure ", + " 'prometheus_metrics' shared dict is present in nginx template") + end + return prometheus +end + +return { + init = init, + init_worker = init_worker, + log = log, + collect = collect, + get_prometheus = get_prometheus, +} diff --git a/images/kong/plugins/0.14.1/prometheus/handler.lua b/images/kong/plugins/0.14.1/prometheus/handler.lua new file mode 100644 index 000000000..0489eed81 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/handler.lua @@ -0,0 +1,24 @@ +local prometheus = require "kong.plugins.prometheus.exporter" +local basic_serializer = require "kong.plugins.log-serializers.basic" + + +prometheus.init() + + +local PrometheusHandler = { + PRIORITY = 13, + VERSION = "0.9.0", +} + +function PrometheusHandler.init_worker() + prometheus.init_worker() +end + + +function PrometheusHandler.log() + local message = basic_serializer.serialize(ngx) + prometheus.log(message) +end + + +return PrometheusHandler diff --git a/images/kong/plugins/0.14.1/prometheus/prometheus.lua b/images/kong/plugins/0.14.1/prometheus/prometheus.lua new file mode 100644 index 000000000..f6b433ae4 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/prometheus.lua @@ -0,0 +1,681 @@ +--- @module Prometheus +-- +-- vim: ts=2:sw=2:sts=2:expandtab:textwidth=80 +-- This module uses a single dictionary shared between Nginx workers to keep +-- all metrics. Each metric is stored as a separate entry in that dictionary. +-- +-- In addition, each worker process has a separate set of counters within +-- its lua runtime that are used to track increments to counte metrics, and +-- are regularly flushed into the main shared dictionary. This is a performance +-- optimization that allows counters to be incremented without locking the +-- shared dictionary. It also means that counter increments are "eventually +-- consistent"; it can take up to a single counter sync interval (which +-- defaults to 1 second) for counter values to be visible for collection. +-- +-- Prometheus requires that (a) all samples for a given metric are presented +-- as one uninterrupted group, and (b) buckets of a histogram appear in +-- increasing numerical order. We satisfy that by carefully constructing full +-- metric names (i.e. metric name along with all labels) so that they meet +-- those requirements while being sorted alphabetically. In particular: +-- +-- * all labels for a given metric are presented in reproducible order (the one +-- used when labels were declared). "le" label for histogram metrics always +-- goes last; +-- * bucket boundaries (which are exposed as values of the "le" label) are +-- presented as floating point numbers with leading and trailing zeroes. +-- Number of of zeroes is determined for each bucketer automatically based on +-- bucket boundaries; +-- * internally "+Inf" bucket is stored as "Inf" (to make it appear after +-- all numeric buckets), and gets replaced by "+Inf" just before we +-- expose the metrics. +-- +-- For example, if you define your bucket boundaries as {0.00005, 10, 1000} +-- then we will keep the following samples for a metric `m1` with label +-- `site` set to `site1`: +-- +-- m1_bucket{site="site1",le="0000.00005"} +-- m1_bucket{site="site1",le="0010.00000"} +-- m1_bucket{site="site1",le="1000.00000"} +-- m1_bucket{site="site1",le="Inf"} +-- m1_count{site="site1"} +-- m1_sum{site="site1"} +-- +-- "Inf" will be replaced by "+Inf" while publishing metrics. +-- +-- You can find the latest version and documentation at +-- https://github.com/knyar/nginx-lua-prometheus +-- Released under MIT license. + +-- This library provides per-worker counters used to store counter metric +-- increments. Copied from https://github.com/Kong/lua-resty-counter +local resty_counter_lib = require("prometheus_resty_counter") + +local Prometheus = {} +local mt = { __index = Prometheus } + +local TYPE_COUNTER = 0x1 +local TYPE_GAUGE = 0x2 +local TYPE_HISTOGRAM = 0x4 +local TYPE_LITERAL = { + [TYPE_COUNTER] = "counter", + [TYPE_GAUGE] = "gauge", + [TYPE_HISTOGRAM] = "histogram", +} + +-- Error metric incremented for this library. +local ERROR_METRIC_NAME = "nginx_metric_errors_total" + +-- Default set of latency buckets, 5ms to 10s: +local DEFAULT_BUCKETS = {0.005, 0.01, 0.02, 0.03, 0.05, 0.075, 0.1, 0.2, 0.3, + 0.4, 0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 10} + +-- Generate full metric name that includes all labels. +-- +-- Args: +-- name: string +-- label_names: (array) a list of label keys. +-- label_values: (array) a list of label values. +-- Returns: +-- (string) full metric name. +local function full_metric_name(name, label_names, label_values) + if not label_names then + return name + end + local label_parts = {} + for idx, key in ipairs(label_names) do + local label_value = (string.format("%s", label_values[idx]) + :gsub("[^\032-\126]", "") -- strip non-printable characters + :gsub("\\", "\\\\") + :gsub('"', '\\"')) + table.insert(label_parts, key .. '="' .. label_value .. '"') + end + return name .. "{" .. table.concat(label_parts, ",") .. "}" +end + +-- Extract short metric name from the full one. +-- +-- This function is only used by Prometheus:metric_data. +-- +-- Args: +-- full_name: (string) full metric name that can include labels. +-- +-- Returns: +-- (string) short metric name with no labels. For a `*_bucket` metric of +-- histogram the _bucket suffix will be removed. +local function short_metric_name(full_name) + local labels_start, _ = full_name:find("{") + if not labels_start then + return full_name + end + -- Try to detect if this is a histogram metric. We only check for the + -- `_bucket` suffix here, since it alphabetically goes before other + -- histogram suffixes (`_count` and `_sum`). + local suffix_idx, _ = full_name:find("_bucket{") + if suffix_idx and full_name:find("le=") then + -- this is a histogram metric + return full_name:sub(1, suffix_idx - 1) + end + -- this is not a histogram metric + return full_name:sub(1, labels_start - 1) +end + +-- Check metric name and label names for correctness. +-- +-- Regular expressions to validate metric and label names are +-- documented in https://prometheus.io/docs/concepts/data_model/ +-- +-- Args: +-- metric_name: (string) metric name. +-- label_names: label names (array of strings). +-- +-- Returns: +-- Either an error string, or nil of no errors were found. +local function check_metric_and_label_names(metric_name, label_names) + if not metric_name:match("^[a-zA-Z_:][a-zA-Z0-9_:]*$") then + return "Metric name '" .. metric_name .. "' is invalid" + end + for _, label_name in ipairs(label_names or {}) do + if label_name == "le" then + return "Invalid label name 'le' in " .. metric_name + end + if not label_name:match("^[a-zA-Z_][a-zA-Z0-9_]*$") then + return "Metric '" .. metric_name .. "' label name '" .. label_name .. + "' is invalid" + end + end +end + +-- Construct bucket format for a list of buckets. +-- +-- This receives a list of buckets and returns a sprintf template that should +-- be used for bucket boundaries to make them come in increasing order when +-- sorted alphabetically. +-- +-- To re-phrase, this is where we detect how many leading and trailing zeros we +-- need. +-- +-- Args: +-- buckets: a list of buckets +-- +-- Returns: +-- (string) a sprintf template. +local function construct_bucket_format(buckets) + local max_order = 1 + local max_precision = 1 + for _, bucket in ipairs(buckets) do + assert(type(bucket) == "number", "bucket boundaries should be numeric") + -- floating point number with all trailing zeros removed + local as_string = string.format("%f", bucket):gsub("0*$", "") + local dot_idx = as_string:find(".", 1, true) + max_order = math.max(max_order, dot_idx - 1) + max_precision = math.max(max_precision, as_string:len() - dot_idx) + end + return "%0" .. (max_order + max_precision + 1) .. "." .. max_precision .. "f" +end + +-- Return a full metric name for a given metric+label combination. +-- +-- This function calculates a full metric name (or, in case of a histogram +-- metric, several metric names) for a given combination of label values. It +-- stores the result in a tree of tables used as a cache (self.lookup) and +-- uses that cache to return results faster. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- label_values: a list of label values. +-- +-- Returns: +-- - If `self` is a counter or a gauge: full metric name as a string. +-- - If `self` is a histogram metric: a list of strings: +-- [0]: full name of the _count histogram metric; +-- [1]: full name of the _sum histogram metric; +-- [...]: full names of each _bucket metrics. +local function lookup_or_create(self, label_values) + -- If one of the `label_values` is nil, #label_values will return the number + -- of non-nil labels in the beginning of the list. This will make us return an + -- error here as well. + local cnt = label_values and #label_values or 0 + -- specially, if first element is nil, # will treat it as "non-empty" + if cnt ~= self.label_count or (self.label_count > 0 and not label_values[1]) then + return nil, string.format("inconsistent labels count, expected %d, got %d", + self.label_count, cnt) + end + local t = self.lookup + if label_values then + -- Don't use ipairs here to avoid inner loop generates trace first + -- Otherwise the inner for loop below is likely to get JIT compiled before + -- the outer loop which include `lookup_or_create`, in this case the trace + -- for outer loop will be aborted. By not using ipairs, we will be able to + -- compile longer traces as possible. + local label + for i=1, self.label_count do + label = label_values[i] + if not t[label] then + t[label] = {} + end + t = t[label] + end + end + + local LEAF_KEY = mt -- key used to store full metric names in leaf tables. + local full_name = t[LEAF_KEY] + if full_name then + return full_name + end + + if self.typ == TYPE_HISTOGRAM then + -- Pass empty metric name to full_metric_name to just get the formatted + -- labels ({key1="value1",key2="value2",...}). + local labels = full_metric_name("", self.label_names, label_values) + full_name = { + self.name .. "_count" .. labels, + self.name .. "_sum" .. labels, + } + + local bucket_pref + if self.label_count > 0 then + -- strip last } + bucket_pref = self.name .. "_bucket" .. string.sub(labels, 1, #labels-1) .. "," + else + bucket_pref = self.name .. "_bucket{" + end + + for i, buc in ipairs(self.buckets) do + full_name[i+2] = string.format("%sle=\"%s\"}", bucket_pref, self.bucket_format:format(buc)) + end + -- Last bucket. Note, that the label value is "Inf" rather than "+Inf" + -- required by Prometheus. This is necessary for this bucket to be the last + -- one when all metrics are lexicographically sorted. "Inf" will get replaced + -- by "+Inf" in Prometheus:metric_data(). + full_name[self.bucket_count+3] = string.format("%sle=\"Inf\"}", bucket_pref) + else + full_name = full_metric_name(self.name, self.label_names, label_values) + end + t[LEAF_KEY] = full_name + return full_name +end + +-- Increment a gauge metric. +-- +-- Gauges are incremented in the dictionary directly to provide strong ordering +-- of inc() and set() operations. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- value: numeric value to increment by. Can be negative. +-- label_values: a list of label values, in the same order as label keys. +local function inc_gauge(self, value, label_values) + local k, err, _ + k, err = lookup_or_create(self, label_values) + if err then + self._log_error(err) + return + end + + _, err, _ = self._dict:incr(k, value, 0) + if err then + self._log_error_kv(k, value, err) + end +end + +local ERR_MSG_COUNTER_NOT_INITIALIZED = "counter not initialied" + +-- Increment a counter metric. +-- +-- Counters are incremented in the per-worker counter, which will eventually get +-- flushed into the global shared dictionary. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- value: numeric value to increment by. Can be negative. +-- label_values: a list of label values, in the same order as label keys. +local function inc_counter(self, value, label_values) + -- counter is not allowed to decrease + if value and value < 0 then + self._log_error_kv(self.name, value, "Value should not be negative") + return + end + + local k, err + k, err = lookup_or_create(self, label_values) + if err then + self._log_error(err) + return + end + + local c = self._counter + if not c then + c = self.parent._counter + if not c then + self._log_error(ERR_MSG_COUNTER_NOT_INITIALIZED) + return + end + self._counter = c + end + c:incr(k, value) +end + +-- Delete a counter or a gauge metric. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- label_values: a list of label values, in the same order as label keys. +local function del(self, label_values) + local k, _, err + k, err = lookup_or_create(self, label_values) + if err then + self._log_error(err) + return + end + + -- Gauge metrics don't use per-worker counters, so for gauges we don't need to wait for + -- the counter to sync. + if self.typ ~= TYPE_GAUGE then + ngx.log(ngx.INFO, "waiting ", self.parent.sync_interval, "s for counter to sync") + ngx.sleep(self.parent.sync_interval) + end + + _, err = self._dict:delete(k) + if err then + self._log_error("Error deleting key: ".. k .. ": " .. err) + end +end + +-- Set the value of a gauge metric. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- value: numeric value. +-- label_values: a list of label values, in the same order as label keys. +local function set(self, value, label_values) + if not value then + self._log_error("No value passed for " .. self.name) + return + end + + local k, _, err + k, err = lookup_or_create(self, label_values) + if err then + self._log_error(err) + return + end + _, err = self._dict:safe_set(k, value) + if err then + self._log_error_kv(k, value, err) + end +end + +-- Record a given value in a histogram. +-- +-- Args: +-- self: a `metric` object, created by register(). +-- value: numeric value to record. Should be defined. +-- label_values: a list of label values, in the same order as label keys. +local function observe(self, value, label_values) + if not value then + self._log_error("No value passed for " .. self.name) + return + end + + local keys, err = lookup_or_create(self, label_values) + if err then + self._log_error(err) + return + end + + local c = self._counter + if not c then + c = self.parent._counter + if not c then + self._log_error(ERR_MSG_COUNTER_NOT_INITIALIZED) + return + end + self._counter = c + end + + -- _count metric. + c:incr(keys[1], 1) + + -- _sum metric. + c:incr(keys[2], value) + + local seen = false + -- check in reverse order, otherwise we will always + -- need to traverse the whole table. + for i=self.bucket_count, 1, -1 do + if value <= self.buckets[i] then + c:incr(keys[2+i], 1) + seen = true + elseif seen then + break + end + end + -- the last bucket (le="Inf"). + c:incr(keys[self.bucket_count+3], 1) +end + +-- Delete all metrics for a given gauge or a counter. +-- +-- This is like `del`, but will delete all time series for all previously +-- recorded label values. +-- +-- Args: +-- self: a `metric` object, created by register(). +local function reset(self) + -- Gauge metrics don't use per-worker counters, so for gauges we don't need to wait for + -- the counter to sync. + if self.typ ~= TYPE_GAUGE then + ngx.log(ngx.INFO, "waiting ", self.parent.sync_interval, "s for counter to sync") + ngx.sleep(self.parent.sync_interval) + end + + local keys = self._dict:get_keys(0) + local name_prefix = self.name .. "{" + local name_prefix_length = #name_prefix + + for _, key in ipairs(keys) do + local value, err = self._dict:get(key) + if value then + -- with out labels equal, or with labels and the part before { equals + if key == self.name or name_prefix == string.sub(key, 1, name_prefix_length) then + _, err = self._dict:safe_set(key, nil) + if err then + self._log_error("Error resetting '", key, "': ", err) + end + end + else + self._log_error("Error getting '", key, "': ", err) + end + end + + -- Clean up the full metric name lookup table as well. + self.lookup = {} +end + +-- Initialize the module. +-- +-- This should be called once from the `init_by_lua` section in nginx +-- configuration. +-- +-- Args: +-- dict_name: (string) name of the nginx shared dictionary which will be +-- used to store all metrics +-- prefix: (optional string) if supplied, prefix is added to all +-- metric names on output +-- +-- Returns: +-- an object that should be used to register metrics. +function Prometheus.init(dict_name, prefix) + local self = setmetatable({}, mt) + dict_name = dict_name or "prometheus_metrics" + self.dict_name = dict_name + self.dict = ngx.shared[dict_name] + if self.dict == nil then + error("Dictionary '" .. dict_name .. "' does not seem to exist. " .. + "Please define the dictionary using `lua_shared_dict`.", 2) + end + + if prefix then + self.prefix = prefix + else + self.prefix = '' + end + + self.registry = {} + + self.initialized = true + + self:counter(ERROR_METRIC_NAME, "Number of nginx-lua-prometheus errors") + self.dict:set(ERROR_METRIC_NAME, 0) + return self +end + +-- Initialize the worker counter. +-- +-- This should be called once from the `init_worker_by_lua` section in nginx +-- configuration. +-- +-- Args: +-- sync_interval: per-worker counter sync interval (in seconds). +function Prometheus:init_worker(sync_interval) + self.sync_interval = sync_interval or 1 + local counter_instance, err = resty_counter_lib.new( + self.dict_name, self.sync_interval) + if err then + error(err, 2) + end + self._counter = counter_instance +end + +-- Register a new metric. +-- +-- Args: +-- self: a Prometheus object. +-- name: (string) name of the metric. Required. +-- help: (string) description of the metric. Will be used for the HELP +-- comment on the metrics page. Optional. +-- label_names: array of strings, defining a list of metrics. Optional. +-- buckets: array if numbers, defining bucket boundaries. Only used for +-- histogram metrics. +-- typ: metric type (one of the TYPE_* constants). +-- +-- Returns: +-- a new metric object. +local function register(self, name, help, label_names, buckets, typ) + if not self.initialized then + ngx.log(ngx.ERR, "Prometheus module has not been initialized") + return + end + + local err = check_metric_and_label_names(name, label_names) + if err then + self:log_error(err) + return + end + + local name_maybe_historgram = name:gsub("_bucket$", "") + :gsub("_count$", "") + :gsub("_sum$", "") + if (self.typ ~= TYPE_HISTOGRAM and ( + self.registry[name] or self.registry[name_maybe_historgram] + )) or + (self.typ == TYPE_HISTOGRAM and ( + self.registry[name] or + self.registry[name .. "_count"] or + self.registry[name .. "_sum"] or self.registry[name .. "_bucket"] + )) then + + self:log_error("Duplicate metric " .. name) + return + end + + local metric = { + name = name, + help = help, + typ = typ, + label_names = label_names, + label_count = label_names and #label_names or 0, + -- Lookup is a tree of lua tables that contain label values, with leaf + -- tables containing full metric names. For example, given a metric + -- `http_count` and labels `host` and `status`, it might contain the + -- following values: + -- ['me.com']['200'][LEAF_KEY] = 'http_count{host="me.com",status="200"}' + -- ['me.com']['500'][LEAF_KEY] = 'http_count{host="me.com",status="500"}' + -- ['my.net']['200'][LEAF_KEY] = 'http_count{host="my.net",status="200"}' + -- ['my.net']['500'][LEAF_KEY] = 'http_count{host="my.net",status="500"}' + lookup = {}, + parent = self, + -- Store a reference for logging functions for faster lookup. + _log_error = function(...) self:log_error(...) end, + _log_error_kv = function(...) self:log_error_kv(...) end, + _dict = self.dict, + } + if typ < TYPE_HISTOGRAM then + if typ == TYPE_GAUGE then + metric.set = set + metric.inc = inc_gauge + else + metric.inc = inc_counter + end + metric.reset = reset + metric.del = del + else + metric.observe = observe + metric.buckets = buckets or DEFAULT_BUCKETS + metric.bucket_count = #metric.buckets + metric.bucket_format = construct_bucket_format(metric.buckets) + end + + self.registry[name] = metric + return metric +end + +-- Public function to register a counter. +function Prometheus:counter(name, help, label_names) + return register(self, name, help, label_names, nil, TYPE_COUNTER) +end + +-- Public function to register a gauge. +function Prometheus:gauge(name, help, label_names) + return register(self, name, help, label_names, nil, TYPE_GAUGE) +end + +-- Public function to register a histogram. +function Prometheus:histogram(name, help, label_names, buckets) + return register(self, name, help, label_names, buckets, TYPE_HISTOGRAM) +end + +-- Prometheus compatible metric data as an array of strings. +-- +-- Returns: +-- Array of strings with all metrics in a text format compatible with +-- Prometheus. +function Prometheus:metric_data() + if not self.initialized then + ngx.log(ngx.ERR, "Prometheus module has not been initialized") + return + end + + -- Force a manual sync of counter local state (mostly to make tests work). + self._counter:sync() + + local keys = self.dict:get_keys(0) + -- Prometheus server expects buckets of a histogram to appear in increasing + -- numerical order of their label values. + table.sort(keys) + + local seen_metrics = {} + local output = {} + for _, key in ipairs(keys) do + local value, err = self.dict:get(key) + if value then + local short_name = short_metric_name(key) + if not seen_metrics[short_name] then + local m = self.registry[short_name] + if m then + if m.help then + table.insert(output, string.format("# HELP %s%s %s\n", + self.prefix, short_name, m.help)) + end + if m.typ then + table.insert(output, string.format("# TYPE %s%s %s\n", + self.prefix, short_name, TYPE_LITERAL[m.typ])) + end + end + seen_metrics[short_name] = true + end + -- Replace "Inf" with "+Inf" in each metric's last bucket 'le' label. + if key:find('le="Inf"', 1, true) then + key = key:gsub('le="Inf"', 'le="+Inf"') + end + table.insert(output, string.format("%s%s %s\n", self.prefix, key, value)) + else + self:log_error("Error getting '", key, "': ", err) + end + end + return output +end + +-- Present all metrics in a text format compatible with Prometheus. +-- +-- This function should be used to expose the metrics on a separate HTTP page. +-- It will get the metrics from the dictionary, sort them, and expose them +-- aling with TYPE and HELP comments. +function Prometheus:collect() + ngx.header.content_type = "text/plain" + ngx.print(self:metric_data()) +end + +-- Log an error, incrementing the error counter. +function Prometheus:log_error(...) + ngx.log(ngx.ERR, ...) + self.dict:incr(ERROR_METRIC_NAME, 1, 0) +end + +-- Log an error that happened while setting up a dictionary key. +function Prometheus:log_error_kv(key, value, err) + self:log_error( + "Error while setting '", key, "' to '", value, "': '", err, "'") +end + +return Prometheus diff --git a/images/kong/plugins/0.14.1/prometheus/schema.lua b/images/kong/plugins/0.14.1/prometheus/schema.lua new file mode 100644 index 000000000..232750062 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/schema.lua @@ -0,0 +1,19 @@ +local function validate_shared_dict() + if not ngx.shared.prometheus_metrics then + return nil, + "ngx shared dict 'prometheus_metrics' not found" + end + return true +end + + +return { + name = "prometheus", + fields = { + { config = { + type = "record", + fields = {}, + custom_validator = validate_shared_dict, + }, }, + }, +} diff --git a/images/kong/plugins/0.14.1/prometheus/serve.lua b/images/kong/plugins/0.14.1/prometheus/serve.lua new file mode 100644 index 000000000..d297a2d92 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/serve.lua @@ -0,0 +1,48 @@ +local lapis = require "lapis" +local prometheus = require "kong.plugins.prometheus.exporter" + + +local kong = kong + + +local app = lapis.Application() + + +app.default_route = function(self) + local path = self.req.parsed_url.path:match("^(.*)/$") + + if path and self.app.router:resolve(path, self) then + return + + elseif self.app.router:resolve(self.req.parsed_url.path .. "/", self) then + return + end + + return self.app.handle_404(self) +end + + +app.handle_404 = function(self) -- luacheck: ignore 212 + local body = '{"message":"Not found"}' + ngx.status = 404 + ngx.header["Content-Type"] = "application/json; charset=utf-8" + ngx.header["Content-Length"] = #body + 1 + ngx.say(body) +end + + +app:match("/", function() + kong.response.exit(200, "Kong Prometheus exporter, visit /metrics") +end) + + +app:match("/metrics", function() + prometheus:collect() +end) + + +return { + prometheus_server = function() + return lapis.serve(app) + end, +} diff --git a/images/kong/plugins/0.14.1/prometheus/status_api.lua b/images/kong/plugins/0.14.1/prometheus/status_api.lua new file mode 100644 index 000000000..9f8cce977 --- /dev/null +++ b/images/kong/plugins/0.14.1/prometheus/status_api.lua @@ -0,0 +1,10 @@ +local prometheus = require "kong.plugins.prometheus.exporter" + + +return { + ["/metrics"] = { + GET = function() + prometheus.collect() + end, + }, +} diff --git a/images/kong/plugins/0.14.1/rate-limiting/policies/init.lua b/images/kong/plugins/0.14.1/rate-limiting/policies/init.lua new file mode 100644 index 000000000..1dbec026a --- /dev/null +++ b/images/kong/plugins/0.14.1/rate-limiting/policies/init.lua @@ -0,0 +1,277 @@ +local singletons = require "kong.singletons" +local timestamp = require "kong.tools.timestamp" +local redis = require "resty.redis" +local policy_cluster = require "kong.plugins.rate-limiting.policies.cluster" +local reports = require "kong.reports" + + +local ngx_log = ngx.log +local shm = ngx.shared.kong_rate_limiting_counters +local pairs = pairs +local fmt = string.format + + +local NULL_UUID = "00000000-0000-0000-0000-000000000000" + + +local function get_ids(conf) + conf = conf or {} + + local api_id = conf.api_id + + if api_id and api_id ~= ngx.null then + return nil, nil, api_id + end + + api_id = NULL_UUID + + local route_id = conf.route_id + local service_id = conf.service_id + + if not route_id or route_id == ngx.null then + route_id = NULL_UUID + end + + if not service_id or service_id == ngx.null then + service_id = NULL_UUID + end + + return route_id, service_id, api_id +end + + +local get_local_key = function(conf, identifier, period_date, name) + local route_id, service_id, api_id = get_ids(conf) + + if api_id == NULL_UUID then + return fmt("ratelimit:%s:%s:%s:%s:%s", route_id, service_id, identifier, period_date, name) + end + + return fmt("ratelimit:%s:%s:%s:%s", api_id, identifier, period_date, name) +end + + +local EXPIRATIONS = { + second = 1, + minute = 60, + hour = 3600, + day = 86400, + month = 2592000, + year = 31536000, +} + + +return { + ["local"] = { + increment = function(conf, limits, identifier, current_timestamp, value) + local periods = timestamp.get_timestamps(current_timestamp) + for period, period_date in pairs(periods) do + if limits[period] then + local cache_key = get_local_key(conf, identifier, period_date, period) + local newval, err = shm:incr(cache_key, value, 0, EXPIRATIONS[period]) + if not newval then + ngx_log(ngx.ERR, "[rate-limiting] could not increment counter ", + "for period '", period, "': ", err) + return nil, err + end + end + end + + return true + end, + usage = function(conf, identifier, current_timestamp, name) + local periods = timestamp.get_timestamps(current_timestamp) + local cache_key = get_local_key(conf, identifier, periods[name], name) + local current_metric, err = shm:get(cache_key) + if err then + return nil, err + end + return current_metric and current_metric or 0 + end + }, + ["cluster"] = { + increment = function(conf, limits, identifier, current_timestamp, value) + local db = singletons.dao.db + local route_id, service_id, api_id = get_ids(conf) + + local ok, err + + if api_id == NULL_UUID then + ok, err = policy_cluster[db.name].increment(db, limits, route_id, service_id, + identifier, current_timestamp, value) + + else + ok, err = policy_cluster[db.name].increment_api(db, limits, api_id, identifier, + current_timestamp, value) + end + + if not ok then + ngx_log(ngx.ERR, "[rate-limiting] cluster policy: could not increment ", + db.name, " counter: ", err) + end + + return ok, err + end, + usage = function(conf, identifier, current_timestamp, name) + local db = singletons.dao.db + local route_id, service_id, api_id = get_ids(conf) + local row, err + + if api_id == NULL_UUID then + row, err = policy_cluster[db.name].find(db, route_id, service_id, + identifier, current_timestamp, name) + else + row, err = policy_cluster[db.name].find_api(db, api_id, identifier, + current_timestamp, name) + end + + if err then + return nil, err + end + + return row and row.value or 0 + end + }, + ["redis"] = { + increment = function(conf, limits, identifier, current_timestamp, value) + local red = redis:new() + red:set_timeout(conf.redis_timeout) + local ok, err = red:connect(conf.redis_host, conf.redis_port) + if not ok then + ngx_log(ngx.ERR, "failed to connect to Redis: ", err) + return nil, err + end + + local times, err = red:get_reused_times() + if err then + ngx_log(ngx.ERR, "failed to get connect reused times: ", err) + return nil, err + end + + if times == 0 and conf.redis_password and conf.redis_password ~= "" then + local ok, err = red:auth(conf.redis_password) + if not ok then + ngx_log(ngx.ERR, "failed to auth Redis: ", err) + return nil, err + end + end + + if times ~= 0 or conf.redis_database then + -- The connection pool is shared between multiple instances of this + -- plugin, and instances of the response-ratelimiting plugin. + -- Because there isn't a way for us to know which Redis database a given + -- socket is connected to without a roundtrip, we force the retrieved + -- socket to select the desired database. + -- When the connection is fresh and the database is the default one, we + -- can skip this roundtrip. + + local ok, err = red:select(conf.redis_database or 0) + if not ok then + ngx_log(ngx.ERR, "failed to change Redis database: ", err) + return nil, err + end + end + + local keys = {} + local expirations = {} + local idx = 0 + local periods = timestamp.get_timestamps(current_timestamp) + for period, period_date in pairs(periods) do + if limits[period] then + local cache_key = get_local_key(conf, identifier, period_date, period) + local exists, err = red:exists(cache_key) + if err then + ngx_log(ngx.ERR, "failed to query Redis: ", err) + return nil, err + end + + idx = idx + 1 + keys[idx] = cache_key + if not exists or exists == 0 then + expirations[idx] = EXPIRATIONS[period] + end + end + end + + red:init_pipeline() + for i = 1, idx do + red:incrby(keys[i], value) + if expirations[i] then + red:expire(keys[i], expirations[i]) + end + end + + local _, err = red:commit_pipeline() + if err then + ngx_log(ngx.ERR, "failed to commit pipeline in Redis: ", err) + return nil, err + end + local ok, err = red:set_keepalive(10000, 100) + if not ok then + ngx_log(ngx.ERR, "failed to set Redis keepalive: ", err) + return nil, err + end + + return true + end, + usage = function(conf, identifier, current_timestamp, name) + local red = redis:new() + red:set_timeout(conf.redis_timeout) + local ok, err = red:connect(conf.redis_host, conf.redis_port) + if not ok then + ngx_log(ngx.ERR, "failed to connect to Redis: ", err) + return nil, err + end + + local times, err = red:get_reused_times() + if err then + ngx_log(ngx.ERR, "failed to get connect reused times: ", err) + return nil, err + end + + if times == 0 and conf.redis_password and conf.redis_password ~= "" then + local ok, err = red:auth(conf.redis_password) + if not ok then + ngx_log(ngx.ERR, "failed to connect to Redis: ", err) + return nil, err + end + end + + if times ~= 0 or conf.redis_database then + -- The connection pool is shared between multiple instances of this + -- plugin, and instances of the response-ratelimiting plugin. + -- Because there isn't a way for us to know which Redis database a given + -- socket is connected to without a roundtrip, we force the retrieved + -- socket to select the desired database. + -- When the connection is fresh and the database is the default one, we + -- can skip this roundtrip. + + local ok, err = red:select(conf.redis_database or 0) + if not ok then + ngx_log(ngx.ERR, "failed to change Redis database: ", err) + return nil, err + end + end + + reports.retrieve_redis_version(red) + + local periods = timestamp.get_timestamps(current_timestamp) + local cache_key = get_local_key(conf, identifier, periods[name], name) + local current_metric, err = red:get(cache_key) + if err then + return nil, err + end + + if current_metric == ngx.null then + current_metric = nil + end + + local ok, err = red:set_keepalive(10000, 100) + if not ok then + ngx_log(ngx.ERR, "failed to set Redis keepalive: ", err) + end + + return current_metric or 0 + end + } +} diff --git a/images/kong/templates/0.14.1/kong_defaults.lua b/images/kong/templates/0.14.1/kong_defaults.lua new file mode 100644 index 000000000..0e9dd060d --- /dev/null +++ b/images/kong/templates/0.14.1/kong_defaults.lua @@ -0,0 +1,80 @@ +return [[ +prefix = /usr/local/kong/ +log_level = notice +proxy_access_log = logs/access.log +proxy_error_log = logs/error.log +admin_access_log = logs/admin_access.log +admin_error_log = logs/error.log +plugins = bundled +custom_plugins = NONE +anonymous_reports = on + +proxy_listen = 0.0.0.0:8000, 0.0.0.0:8443 ssl +admin_listen = 127.0.0.1:8001, 127.0.0.1:8444 ssl +nginx_user = nobody nobody +nginx_worker_processes = auto +nginx_optimizations = on +nginx_daemon = on +mem_cache_size = 128m +ratelimit_cache_size = 12m +ssl_cert = NONE +ssl_cert_key = NONE +client_ssl = off +client_ssl_cert = NONE +client_ssl_cert_key = NONE +ssl_cipher_suite = modern +ssl_ciphers = ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 +admin_ssl_cert = NONE +admin_ssl_cert_key = NONE +upstream_keepalive = 60 +headers = server_tokens, latency_tokens +trusted_ips = NONE +real_ip_header = X-Real-IP +real_ip_recursive = off +client_max_body_size = 0 +client_body_buffer_size = 8k +error_default_type = text/plain + +database = postgres +pg_host = 127.0.0.1 +pg_port = 5432 +pg_database = kong +pg_user = kong +pg_password = NONE +pg_ssl = off +pg_ssl_verify = off +cassandra_contact_points = 127.0.0.1 +cassandra_port = 9042 +cassandra_keyspace = kong +cassandra_timeout = 5000 +cassandra_ssl = off +cassandra_ssl_verify = off +cassandra_username = kong +cassandra_password = NONE +cassandra_consistency = ONE +cassandra_lb_policy = RoundRobin +cassandra_local_datacenter = NONE +cassandra_repl_strategy = SimpleStrategy +cassandra_repl_factor = 1 +cassandra_data_centers = dc1:2,dc2:3 +cassandra_schema_consensus_timeout = 10000 + +db_update_frequency = 5 +db_update_propagation = 0 +db_cache_ttl = 0 +db_resurrect_ttl = 30 + +dns_resolver = NONE +dns_hostsfile = /etc/hosts +dns_order = LAST,SRV,A,CNAME +dns_stale_ttl = 4 +dns_not_found_ttl = 30 +dns_error_ttl = 1 +dns_no_sync = off + +lua_socket_pool_size = 30 +lua_ssl_trusted_certificate = NONE +lua_ssl_verify_depth = 1 +lua_package_path = ./?.lua;./?/init.lua; +lua_package_cpath = NONE +]] diff --git a/images/kong/templates/0.14.1/nginx_kong.lua b/images/kong/templates/0.14.1/nginx_kong.lua new file mode 100644 index 000000000..0fb2d2fbe --- /dev/null +++ b/images/kong/templates/0.14.1/nginx_kong.lua @@ -0,0 +1,234 @@ +return [[ +charset UTF-8; + +> if anonymous_reports then +${{SYSLOG_REPORTS}} +> end + +error_log ${{PROXY_ERROR_LOG}} ${{LOG_LEVEL}}; + +> if nginx_optimizations then +>-- send_timeout 60s; # default value +>-- keepalive_timeout 75s; # default value +>-- client_body_timeout 60s; # default value +>-- client_header_timeout 60s; # default value +>-- tcp_nopush on; # disabled until benchmarked +>-- proxy_buffer_size 128k; # disabled until benchmarked +>-- proxy_buffers 4 256k; # disabled until benchmarked +>-- proxy_busy_buffers_size 256k; # disabled until benchmarked +>-- reset_timedout_connection on; # disabled until benchmarked +> end + +client_max_body_size ${{CLIENT_MAX_BODY_SIZE}}; +proxy_ssl_server_name on; +underscores_in_headers on; + +lua_package_path '${{LUA_PACKAGE_PATH}};;'; +lua_package_cpath '${{LUA_PACKAGE_CPATH}};;'; +lua_socket_pool_size ${{LUA_SOCKET_POOL_SIZE}}; +lua_max_running_timers 4096; +lua_max_pending_timers 16384; +lua_shared_dict kong 5m; +lua_shared_dict kong_db_cache ${{MEM_CACHE_SIZE}}; +lua_shared_dict kong_db_cache_miss 12m; +lua_shared_dict kong_locks 8m; +lua_shared_dict kong_process_events 5m; +lua_shared_dict kong_cluster_events 5m; +lua_shared_dict kong_healthchecks 5m; +lua_shared_dict kong_rate_limiting_counters ${{RATELIMIT_CACHE_SIZE}}; +> if database == "cassandra" then +lua_shared_dict kong_cassandra 5m; +> end +lua_socket_log_errors off; +> if lua_ssl_trusted_certificate then +lua_ssl_trusted_certificate '${{LUA_SSL_TRUSTED_CERTIFICATE}}'; +lua_ssl_verify_depth ${{LUA_SSL_VERIFY_DEPTH}}; +> end + +# injected nginx_http_* directives +> for _, el in ipairs(nginx_http_directives) do +$(el.name) $(el.value); +> end + +init_by_lua_block { + Kong = require 'kong' + Kong.init() +} + +init_worker_by_lua_block { + Kong.init_worker() +} + + +> if #proxy_listeners > 0 then +upstream kong_upstream { + server 0.0.0.1; + balancer_by_lua_block { + Kong.balancer() + } + keepalive ${{UPSTREAM_KEEPALIVE}}; +} + +server { + server_name kong; +> for i = 1, #proxy_listeners do + listen $(proxy_listeners[i].listener); +> end + error_page 400 404 408 411 412 413 414 417 494 /kong_error_handler; + error_page 500 502 503 504 /kong_error_handler; + + access_log ${{PROXY_ACCESS_LOG}}; + error_log ${{PROXY_ERROR_LOG}} ${{LOG_LEVEL}}; + + client_body_buffer_size ${{CLIENT_BODY_BUFFER_SIZE}}; + +> if proxy_ssl_enabled then + ssl_certificate ${{SSL_CERT}}; + ssl_certificate_key ${{SSL_CERT_KEY}}; + ssl_protocols TLSv1.1 TLSv1.2; + ssl_certificate_by_lua_block { + Kong.ssl_certificate() + } + + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + ssl_prefer_server_ciphers on; + ssl_ciphers ${{SSL_CIPHERS}}; +> end + +> if client_ssl then + proxy_ssl_certificate ${{CLIENT_SSL_CERT}}; + proxy_ssl_certificate_key ${{CLIENT_SSL_CERT_KEY}}; +> end + + real_ip_header ${{REAL_IP_HEADER}}; + real_ip_recursive ${{REAL_IP_RECURSIVE}}; +> for i = 1, #trusted_ips do + set_real_ip_from $(trusted_ips[i]); +> end + + # injected nginx_proxy_* directives +> for _, el in ipairs(nginx_proxy_directives) do + $(el.name) $(el.value); +> end + + location / { + default_type ''; + + set $ctx_ref ''; + set $upstream_host ''; + set $upstream_upgrade ''; + set $upstream_connection ''; + set $upstream_scheme ''; + set $upstream_uri ''; + set $upstream_x_forwarded_for ''; + set $upstream_x_forwarded_proto ''; + set $upstream_x_forwarded_host ''; + set $upstream_x_forwarded_port ''; + + rewrite_by_lua_block { + Kong.rewrite() + } + + access_by_lua_block { + Kong.access() + } + + proxy_http_version 1.1; + proxy_set_header Host $upstream_host; + proxy_set_header Upgrade $upstream_upgrade; + proxy_set_header Connection $upstream_connection; + proxy_set_header X-Forwarded-For $upstream_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $upstream_x_forwarded_proto; + proxy_set_header X-Forwarded-Host $upstream_x_forwarded_host; + proxy_set_header X-Forwarded-Port $upstream_x_forwarded_port; + proxy_set_header X-Real-IP $remote_addr; + proxy_pass_header Server; + proxy_pass_header Date; + proxy_ssl_name $upstream_host; + proxy_pass $upstream_scheme://kong_upstream$upstream_uri; + + header_filter_by_lua_block { + Kong.header_filter() + } + + body_filter_by_lua_block { + Kong.body_filter() + } + + log_by_lua_block { + Kong.log() + } + } + + location = /kong_error_handler { + internal; + uninitialized_variable_warn off; + + content_by_lua_block { + Kong.handle_error() + } + + header_filter_by_lua_block { + Kong.header_filter() + } + + body_filter_by_lua_block { + Kong.body_filter() + } + + log_by_lua_block { + Kong.log() + } + } +} +> end + +> if #admin_listeners > 0 then +server { + server_name kong_admin; +> for i = 1, #admin_listeners do + listen $(admin_listeners[i].listener); +> end + + access_log ${{ADMIN_ACCESS_LOG}}; + error_log ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}}; + + client_max_body_size 10m; + client_body_buffer_size 10m; + +> if admin_ssl_enabled then + ssl_certificate ${{ADMIN_SSL_CERT}}; + ssl_certificate_key ${{ADMIN_SSL_CERT_KEY}}; + ssl_protocols TLSv1.1 TLSv1.2; + + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + ssl_prefer_server_ciphers on; + ssl_ciphers ${{SSL_CIPHERS}}; +> end + + # injected nginx_admin_* directives +> for _, el in ipairs(nginx_admin_directives) do + $(el.name) $(el.value); +> end + + location / { + default_type application/json; + content_by_lua_block { + Kong.serve_admin_api() + } + } + + location /nginx_status { + internal; + access_log off; + stub_status; + } + + location /robots.txt { + return 200 'User-agent: *\nDisallow: /'; + } +} +> end +]] diff --git a/images/openbadger/Dockerfile b/images/openbadger/Dockerfile index 44a30c92c..40a2d4721 100755 --- a/images/openbadger/Dockerfile +++ b/images/openbadger/Dockerfile @@ -1,7 +1,8 @@ -FROM ubuntu:17.10 +FROM ubuntu:18.04 MAINTAINER Rajesh Rajendran # Installing and configuring packages +RUN apt update && apt install -y python-psycopg2 COPY ./images/openbadger/configure.sh /tmp RUN /tmp/configure.sh diff --git a/images/proxy/Dockerfile b/images/proxy/Dockerfile index 7bdb2cae3..da74f9731 100644 --- a/images/proxy/Dockerfile +++ b/images/proxy/Dockerfile @@ -10,7 +10,9 @@ ENV LUAJIT_INC=/usr/local/include/luajit-2.1 COPY nginx_devel_kit /opt/nginx_devel_kit COPY luajit /usr/src/luajit COPY nginx_lua /opt/nginx_lua +COPY ngx_http_geoip2_module /opt/nginx_http_geoip2_module +RUN apk add --no-cache libmaxminddb-dev RUN GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \ && CONFIG="\ --prefix=/etc/nginx \ @@ -19,6 +21,7 @@ RUN GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \ --with-ld-opt='-Wl,-rpath,/usr/local/lib' \ --add-module=/opt/nginx_devel_kit \ --add-module=/opt/nginx_lua \ + --add-dynamic-module=/opt/nginx_http_geoip2_module \ --conf-path=/etc/nginx/nginx.conf \ --error-log-path=/var/log/nginx/error.log \ --http-log-path=/var/log/nginx/access.log \ @@ -154,7 +157,7 @@ RUN GPG_KEYS=B0F4253373F8F6F510D42178520A9993A1C052F8 \ && ln -sf /dev/stderr /var/log/nginx/error.log COPY nginx.conf /etc/nginx/nginx.conf -copy prometheus.lua /etc/nginx/lua_modules/ +copy nginx-lua-prometheus/*.lua /etc/nginx/lua_modules/ EXPOSE 80 diff --git a/images/proxy/Jenkinsfile b/images/proxy/Jenkinsfile index 1d80a01f5..8b2839365 100644 --- a/images/proxy/Jenkinsfile +++ b/images/proxy/Jenkinsfile @@ -17,14 +17,16 @@ node('build-slave') { cleanWs() if (params.github_release_tag == "") { checkout scm + sh("git submodule update --init --recursive") commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim() branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim() - build_tag = branch_name + "_" + commit_hash + build_tag = branch_name + "_" + commit_hash + "_" + env.BUILD_NUMBER println(ANSI_BOLD + ANSI_YELLOW + "Tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL) } else { def scmVars = checkout scm checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]] - build_tag = params.github_release_tag + sh("git submodule update --init --recursive") + build_tag = params.github_release_tag + "_" + env.BUILD_NUMBER println(ANSI_BOLD + ANSI_YELLOW + "Tag specified, building from tag: " + params.github_release_tag + ANSI_NORMAL) } echo "build_tag: " + build_tag diff --git a/images/proxy/build.sh b/images/proxy/build.sh index b7d6a3c9f..738938f80 100755 --- a/images/proxy/build.sh +++ b/images/proxy/build.sh @@ -9,7 +9,7 @@ wget https://codeload.github.com/openresty/luajit2/tar.gz/v2.1-20190626 -O luaji wget https://codeload.github.com/openresty/lua-nginx-module/tar.gz/v0.10.15 -O ngx_lua.tar.gz # Creating deps directory -mkdir nginx_devel_kit luajit nginx_lua +mkdir -p nginx_devel_kit luajit nginx_lua tar --strip-components=1 -xf ngx_devel_kit_0_3_0.tar.gz -C nginx_devel_kit tar --strip-components=1 -xf luajit_2_1.tar.gz -C luajit tar --strip-components=1 -xf ngx_lua.tar.gz -C nginx_lua diff --git a/images/proxy/nginx-lua-prometheus b/images/proxy/nginx-lua-prometheus new file mode 160000 index 000000000..71ac90826 --- /dev/null +++ b/images/proxy/nginx-lua-prometheus @@ -0,0 +1 @@ +Subproject commit 71ac9082637ff1269f195e4f4f5e61836782b8d0 diff --git a/images/proxy/nginx.conf b/images/proxy/nginx.conf index dfdcc8d79..fcf5621b7 100644 --- a/images/proxy/nginx.conf +++ b/images/proxy/nginx.conf @@ -1,14 +1,11 @@ user nginx; worker_processes 1; -load_module modules/ndk_http_module.so; -load_module modules/ngx_http_lua_module.so; - error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { - worker_connections 1024; + worker_connections 10000; } @@ -26,45 +23,61 @@ http { access_log /var/log/nginx/access.log main; # Shared dictionary to store metrics - lua_shared_dict prometheus_metrics 10M; + lua_shared_dict prometheus_metrics 100M; lua_package_path "/etc/nginx/lua_modules/?.lua"; - # Collecting metrics - init_by_lua ' + + # Defining upstream cache status for nginx metrics + map $upstream_cache_status $cache_status { + default $upstream_cache_status; + '' "NONE"; + } + + # Defining metrics + init_worker_by_lua_block { prometheus = require("prometheus").init("prometheus_metrics") metric_requests = prometheus:counter( - "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method"}) + "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method", "cache_status"}) metric_latency = prometheus:histogram( "nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) metric_connections = prometheus:gauge( "nginx_http_connections", "Number of HTTP connections", {"state"}) - '; - - # Collecting metrics - log_by_lua ' - metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method}) + } + log_by_lua_block { + metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method, ngx.var.cache_status }) metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name}) - '; + } + + # local caching for images and files + proxy_cache_path /tmp/proxy_cache levels=1:2 keys_zone=proxy_cache:5m max_size=10m inactive=60m use_temp_path=off; + + # cache framework + proxy_cache_path /tmp/framework_cache levels=1:2 keys_zone=framework_cache:5m max_size=700m inactive=60m use_temp_path=off; + header_filter_by_lua_block { + ngx.header["server"] = nil + } + sendfile on; #tcp_nopush on; + client_max_body_size 60M; - keepalive_timeout 500s; + keepalive_timeout 65s; keepalive_requests 200; #gzip on; include /etc/nginx/conf.d/*.conf; - server { - listen 9145; - location /metrics { - content_by_lua ' + server { + listen 9145; + location /metrics { + content_by_lua_block { metric_connections:set(ngx.var.connections_reading, {"reading"}) metric_connections:set(ngx.var.connections_waiting, {"waiting"}) metric_connections:set(ngx.var.connections_writing, {"writing"}) prometheus:collect() - '; - } - } + } + } + } } diff --git a/images/proxy/ngx_http_geoip2_module/LICENSE b/images/proxy/ngx_http_geoip2_module/LICENSE new file mode 100644 index 000000000..fdc13a7f2 --- /dev/null +++ b/images/proxy/ngx_http_geoip2_module/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2014, Lee Valentine +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/images/proxy/ngx_http_geoip2_module/README.md b/images/proxy/ngx_http_geoip2_module/README.md new file mode 100644 index 000000000..d3b62c165 --- /dev/null +++ b/images/proxy/ngx_http_geoip2_module/README.md @@ -0,0 +1,130 @@ +Description +=========== + +**ngx_http_geoip2_module** - creates variables with values from the maxmind geoip2 databases based on the client IP (default) or from a specific variable (supports both IPv4 and IPv6) + +The module now supports nginx streams and can be used in the same way the http module can be used. + +## Installing +First install [libmaxminddb](https://github.com/maxmind/libmaxminddb) as described in its [README.md +file](https://github.com/maxmind/libmaxminddb/blob/master/README.md#installing-from-a-tarball). + +#### Download nginx source +``` +wget http://nginx.org/download/nginx-VERSION.tar.gz +tar zxvf nginx-VERSION.tar.gz +cd nginx-VERSION +``` + +##### To build as a dynamic module (nginx 1.9.11+): +``` +./configure --add-dynamic-module=/path/to/ngx_http_geoip2_module +make +make install +``` + +This will produce ```objs/ngx_http_geoip2_module.so```. It can be copied to your nginx module path manually if you wish. + +Add the following line to your nginx.conf: +``` +load_module modules/ngx_http_geoip2_module.so; +``` + +##### To build as a static module: +``` +./configure --add-module=/path/to/ngx_http_geoip2_module +make +make install +``` + +## Download Maxmind GeoLite2 Database (optional) +The free GeoLite2 databases are available from [Maxminds website](http://dev.maxmind.com/geoip/geoip2/geolite2/) (requires signing up) + +## Example Usage: +``` +http { + ... + geoip2 /etc/maxmind-country.mmdb { + auto_reload 5m; + $geoip2_metadata_country_build metadata build_epoch; + $geoip2_data_country_code default=US source=$variable_with_ip country iso_code; + $geoip2_data_country_name country names en; + } + + geoip2 /etc/maxmind-city.mmdb { + $geoip2_data_city_name default=London city names en; + } + .... + + fastcgi_param COUNTRY_CODE $geoip2_data_country_code; + fastcgi_param COUNTRY_NAME $geoip2_data_country_name; + fastcgi_param CITY_NAME $geoip2_data_city_name; + .... +} + +stream { + ... + geoip2 /etc/maxmind-country.mmdb { + $geoip2_data_country_code default=US source=$remote_addr country iso_code; + } + ... +} +``` + +##### Metadata: +Retrieve metadata regarding the geoip database. +``` +$variable_name metadata +``` +Available fields: + - build_epoch: the build timestamp of the maxmind database. + - last_check: the last time the database was checked for changes (when using auto_reload) + - last_change: the last time the database was reloaded (when using auto_reload) + +##### Autoreload (default: disabled): +Enabling auto reload will have nginx check the modification time of the database at the specified +interval and reload it if it has changed. +``` +auto_reload +``` + +##### GeoIP: +``` +$variable_name [default= + "iso_code": + "US" + "names": + { + "de": + "USA" + "en": + "United States" + } + } + } + +$ mmdblookup --file /usr/share/GeoIP/GeoIP2-Country.mmdb --ip 8.8.8.8 country names en + + "United States" +``` + +This translates to: + +``` +$country_name "default=United States" source=$remote_addr country names en +``` diff --git a/images/proxy/ngx_http_geoip2_module/config b/images/proxy/ngx_http_geoip2_module/config new file mode 100644 index 000000000..48bf15d35 --- /dev/null +++ b/images/proxy/ngx_http_geoip2_module/config @@ -0,0 +1,43 @@ +ngx_feature="MaxmindDB library" +ngx_feature_name= +ngx_feature_run=no +ngx_feature_incs="#include " +ngx_feature_libs=-lmaxminddb +ngx_feature_test="MMDB_s mmdb" +. auto/feature + +ngx_addon_name="ngx_geoip2_module" + +if [ $ngx_found = yes ]; then + if test -n "$ngx_module_link"; then + if [ $HTTP != NO ]; then + ngx_module_type=HTTP + ngx_module_name="ngx_http_geoip2_module" + ngx_module_incs= + ngx_module_deps= + ngx_module_srcs="$ngx_addon_dir/ngx_http_geoip2_module.c" + ngx_module_libs="$ngx_feature_libs" + . auto/module + fi + + nginx_version=`awk '/^#define nginx_version / {print $3}' src/core/nginx.h` + if [ $STREAM != NO -a $nginx_version -gt 1011001 ]; then + ngx_module_type=STREAM + ngx_module_name="ngx_stream_geoip2_module" + ngx_module_incs= + ngx_module_deps= + ngx_module_srcs="$ngx_addon_dir/ngx_stream_geoip2_module.c" + ngx_module_libs="$ngx_feature_libs" + . auto/module + fi + else + HTTP_MODULES="$HTTP_MODULES ngx_http_geoip2_module" + NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_geoip2_module.c" + CORE_LIBS="$CORE_LIBS $ngx_feature_libs" + fi +else + cat << END +$0: error: the geoip2 module requires the maxminddb library. +END + exit 1 +fi diff --git a/images/proxy/ngx_http_geoip2_module/ngx_http_geoip2_module.c b/images/proxy/ngx_http_geoip2_module/ngx_http_geoip2_module.c new file mode 100644 index 000000000..d27c94d47 --- /dev/null +++ b/images/proxy/ngx_http_geoip2_module/ngx_http_geoip2_module.c @@ -0,0 +1,793 @@ +/* + * Copyright (C) Lee Valentine + * + * Based on nginx's 'ngx_http_geoip_module.c' by Igor Sysoev + */ + + +#include +#include +#include + +#include + + +typedef struct { + MMDB_s mmdb; + MMDB_lookup_result_s result; + time_t last_check; + time_t last_change; + time_t check_interval; +#if (NGX_HAVE_INET6) + uint8_t address[16]; +#else + unsigned long address; +#endif + ngx_queue_t queue; +} ngx_http_geoip2_db_t; + +typedef struct { + ngx_queue_t databases; + ngx_array_t *proxies; + ngx_flag_t proxy_recursive; +} ngx_http_geoip2_conf_t; + +typedef struct { + ngx_http_geoip2_db_t *database; + const char **lookup; + ngx_str_t default_value; + ngx_http_complex_value_t source; +} ngx_http_geoip2_ctx_t; + +typedef struct { + ngx_http_geoip2_db_t *database; + ngx_str_t metavalue; +} ngx_http_geoip2_metadata_t; + + +static ngx_int_t ngx_http_geoip2_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_geoip2_metadata(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); +static void *ngx_http_geoip2_create_conf(ngx_conf_t *cf); +static char *ngx_http_geoip2_init_conf(ngx_conf_t *cf, void *conf); +static char *ngx_http_geoip2(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_http_geoip2_parse_config(ngx_conf_t *cf, ngx_command_t *dummy, + void *conf); +static char *ngx_http_geoip2_add_variable(ngx_conf_t *cf, ngx_command_t *dummy, + void *conf); +static char *ngx_http_geoip2_add_variable_geodata(ngx_conf_t *cf, + ngx_http_geoip2_db_t *database); +static char *ngx_http_geoip2_add_variable_metadata(ngx_conf_t *cf, + ngx_http_geoip2_db_t *database); +static char *ngx_http_geoip2_proxy(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static ngx_int_t ngx_http_geoip2_cidr_value(ngx_conf_t *cf, ngx_str_t *net, + ngx_cidr_t *cidr); +static void ngx_http_geoip2_cleanup(void *data); +static ngx_int_t ngx_http_geoip2_init(ngx_conf_t *cf); + + +#define FORMAT(fmt, ...) do { \ + p = ngx_palloc(r->pool, NGX_OFF_T_LEN); \ + if (p == NULL) { \ + return NGX_ERROR; \ + } \ + v->len = ngx_sprintf(p, fmt, __VA_ARGS__) - p; \ + v->data = p; \ +} while (0) + +static ngx_command_t ngx_http_geoip2_commands[] = { + + { ngx_string("geoip2"), + NGX_HTTP_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_TAKE1, + ngx_http_geoip2, + NGX_HTTP_MAIN_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("geoip2_proxy"), + NGX_HTTP_MAIN_CONF|NGX_CONF_TAKE1, + ngx_http_geoip2_proxy, + NGX_HTTP_MAIN_CONF_OFFSET, + 0, + NULL }, + + { ngx_string("geoip2_proxy_recursive"), + NGX_HTTP_MAIN_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_MAIN_CONF_OFFSET, + offsetof(ngx_http_geoip2_conf_t, proxy_recursive), + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_geoip2_module_ctx = { + NULL, /* preconfiguration */ + ngx_http_geoip2_init, /* postconfiguration */ + + ngx_http_geoip2_create_conf, /* create main configuration */ + ngx_http_geoip2_init_conf, /* init main configuration */ + + NULL, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_geoip2_module = { + NGX_MODULE_V1, + &ngx_http_geoip2_module_ctx, /* module context */ + ngx_http_geoip2_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_http_geoip2_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, + uintptr_t data) +{ + ngx_http_geoip2_ctx_t *geoip2 = (ngx_http_geoip2_ctx_t *) data; + ngx_http_geoip2_db_t *database = geoip2->database; + int mmdb_error; + MMDB_entry_data_s entry_data; + ngx_http_geoip2_conf_t *gcf; + ngx_addr_t addr; + ngx_array_t *xfwd; + u_char *p; + ngx_str_t val; + +#if (NGX_HAVE_INET6) + uint8_t address[16], *addressp = address; +#else + unsigned long address; +#endif + + if (geoip2->source.value.len > 0) { + if (ngx_http_complex_value(r, &geoip2->source, &val) != NGX_OK) { + goto not_found; + } + + if (ngx_parse_addr(r->pool, &addr, val.data, val.len) != NGX_OK) { + goto not_found; + } + } else { + gcf = ngx_http_get_module_main_conf(r, ngx_http_geoip2_module); + addr.sockaddr = r->connection->sockaddr; + addr.socklen = r->connection->socklen; + + xfwd = &r->headers_in.x_forwarded_for; + + if (xfwd->nelts > 0 && gcf->proxies != NULL) { + (void) ngx_http_get_forwarded_addr(r, &addr, xfwd, NULL, + gcf->proxies, gcf->proxy_recursive); + } + } + + switch (addr.sockaddr->sa_family) { + case AF_INET: +#if (NGX_HAVE_INET6) + ngx_memset(addressp, 0, 12); + ngx_memcpy(addressp + 12, &((struct sockaddr_in *) + addr.sockaddr)->sin_addr.s_addr, 4); + break; + + case AF_INET6: + ngx_memcpy(addressp, &((struct sockaddr_in6 *) + addr.sockaddr)->sin6_addr.s6_addr, 16); +#else + address = ((struct sockaddr_in *)addr.sockaddr)->sin_addr.s_addr; +#endif + break; + + default: + goto not_found; + } + +#if (NGX_HAVE_INET6) + if (ngx_memcmp(&address, &database->address, sizeof(address)) + != 0) { +#else + if (address != database->address) { +#endif + memcpy(&database->address, &address, sizeof(address)); + database->result = MMDB_lookup_sockaddr(&database->mmdb, + addr.sockaddr, &mmdb_error); + + if (mmdb_error != MMDB_SUCCESS) { + goto not_found; + } + } + + if (!database->result.found_entry + || MMDB_aget_value(&database->result.entry, &entry_data, + geoip2->lookup) != MMDB_SUCCESS) { + goto not_found; + } + + if (!entry_data.has_data) { + goto not_found; + } + + switch (entry_data.type) { + case MMDB_DATA_TYPE_BOOLEAN: + FORMAT("%d", entry_data.boolean); + break; + case MMDB_DATA_TYPE_UTF8_STRING: + v->len = entry_data.data_size; + v->data = ngx_pnalloc(r->pool, v->len); + if (v->data == NULL) { + return NGX_ERROR; + } + ngx_memcpy(v->data, (u_char *) entry_data.utf8_string, v->len); + break; + case MMDB_DATA_TYPE_BYTES: + v->len = entry_data.data_size; + v->data = ngx_pnalloc(r->pool, v->len); + if (v->data == NULL) { + return NGX_ERROR; + } + ngx_memcpy(v->data, (u_char *) entry_data.bytes, v->len); + break; + case MMDB_DATA_TYPE_FLOAT: + FORMAT("%.5f", entry_data.float_value); + break; + case MMDB_DATA_TYPE_DOUBLE: + FORMAT("%.5f", entry_data.double_value); + break; + case MMDB_DATA_TYPE_UINT16: + FORMAT("%uD", entry_data.uint16); + break; + case MMDB_DATA_TYPE_UINT32: + FORMAT("%uD", entry_data.uint32); + break; + case MMDB_DATA_TYPE_INT32: + FORMAT("%D", entry_data.int32); + break; + case MMDB_DATA_TYPE_UINT64: + FORMAT("%uL", entry_data.uint64); + break; + case MMDB_DATA_TYPE_UINT128: ; +#if MMDB_UINT128_IS_BYTE_ARRAY + uint8_t *val = (uint8_t *)entry_data.uint128; + FORMAT( "0x%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x", + val[0], val[1], val[2], val[3], + val[4], val[5], val[6], val[7], + val[8], val[9], val[10], val[11], + val[12], val[13], val[14], val[15]); +#else + mmdb_uint128_t val = entry_data.uint128; + FORMAT("0x%016uxL%016uxL", + (uint64_t) (val >> 64), (uint64_t) val); +#endif + break; + default: + goto not_found; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; + +not_found: + if (geoip2->default_value.len > 0) { + v->data = geoip2->default_value.data; + v->len = geoip2->default_value.len; + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + } else { + v->not_found = 1; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_geoip2_metadata(ngx_http_request_t *r, ngx_http_variable_value_t *v, + uintptr_t data) +{ + ngx_http_geoip2_metadata_t *metadata = (ngx_http_geoip2_metadata_t *) data; + ngx_http_geoip2_db_t *database = metadata->database; + u_char *p; + + if (ngx_strncmp(metadata->metavalue.data, "build_epoch", 11) == 0) { + FORMAT("%uL", database->mmdb.metadata.build_epoch); + } else if (ngx_strncmp(metadata->metavalue.data, "last_check", 10) == 0) { + FORMAT("%T", database->last_check); + } else if (ngx_strncmp(metadata->metavalue.data, "last_change", 11) == 0) { + FORMAT("%T", database->last_change); + } else { + v->not_found = 1; + return NGX_OK; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; +} + + +static void * +ngx_http_geoip2_create_conf(ngx_conf_t *cf) +{ + ngx_pool_cleanup_t *cln; + ngx_http_geoip2_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, sizeof(ngx_http_geoip2_conf_t)); + if (conf == NULL) { + return NULL; + } + + conf->proxy_recursive = NGX_CONF_UNSET; + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NULL; + } + + ngx_queue_init(&conf->databases); + + cln->handler = ngx_http_geoip2_cleanup; + cln->data = conf; + + return conf; +} + + +static char * +ngx_http_geoip2(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_geoip2_conf_t *gcf = conf; + ngx_str_t *value; + int status; + ngx_http_geoip2_db_t *database; + char *rv; + ngx_conf_t save; + ngx_queue_t *q; + + value = cf->args->elts; + + if (value[1].data && value[1].data[0] != '/') { + if (ngx_conf_full_name(cf->cycle, &value[1], 0) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + + if (!ngx_queue_empty(&gcf->databases)) { + for (q = ngx_queue_head(&gcf->databases); + q != ngx_queue_sentinel(&gcf->databases); + q = ngx_queue_next(q)) + { + database = ngx_queue_data(q, ngx_http_geoip2_db_t, queue); + if (ngx_strcmp(value[1].data, database->mmdb.filename) == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Duplicate GeoIP2 mmdb - %V", &value[1]); + return NGX_CONF_ERROR; + } + } + } + + database = ngx_pcalloc(cf->pool, sizeof(ngx_http_geoip2_db_t)); + if (database == NULL) { + return NGX_CONF_ERROR; + } + + ngx_queue_insert_tail(&gcf->databases, &database->queue); + database->last_check = database->last_change = ngx_time(); + + status = MMDB_open((char *) value[1].data, MMDB_MODE_MMAP, &database->mmdb); + + if (status != MMDB_SUCCESS) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "MMDB_open(\"%V\") failed - %s", &value[1], + MMDB_strerror(status)); + return NGX_CONF_ERROR; + } + + save = *cf; + cf->handler = ngx_http_geoip2_parse_config; + cf->handler_conf = (void *) database; + + rv = ngx_conf_parse(cf, NULL); + *cf = save; + return rv; +} + + +static char * +ngx_http_geoip2_parse_config(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) +{ + ngx_http_geoip2_db_t *database; + ngx_str_t *value; + time_t interval; + + value = cf->args->elts; + + if (value[0].data[0] == '$') { + return ngx_http_geoip2_add_variable(cf, dummy, conf); + } + + if (value[0].len == 11 + && ngx_strncmp(value[0].data, "auto_reload", 11) == 0) { + if ((int) cf->args->nelts != 2) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid number of arguments for auto_reload"); + return NGX_CONF_ERROR; + } + + interval = ngx_parse_time(&value[1], true); + + if (interval == (time_t) NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid interval for auto_reload \"%V\"", + value[1]); + return NGX_CONF_ERROR; + } + + + database = (ngx_http_geoip2_db_t *) conf; + database->check_interval = interval; + return NGX_CONF_OK; + } + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid setting \"%V\"", &value[0]); + return NGX_CONF_ERROR; +} + + +static char * +ngx_http_geoip2_add_variable(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) +{ + ngx_http_geoip2_db_t *database; + ngx_str_t *value; + int nelts; + + value = cf->args->elts; + + if (value[0].data[0] != '$') { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid variable name \"%V\"", &value[0]); + return NGX_CONF_ERROR; + } + + value[0].len--; + value[0].data++; + + nelts = (int) cf->args->nelts; + database = (ngx_http_geoip2_db_t *) conf; + + if (nelts > 0 && value[1].len == 8 && ngx_strncmp(value[1].data, "metadata", 8) == 0) { + return ngx_http_geoip2_add_variable_metadata(cf, database); + } + + return ngx_http_geoip2_add_variable_geodata(cf, database); +} + + +static char * +ngx_http_geoip2_add_variable_metadata(ngx_conf_t *cf, ngx_http_geoip2_db_t *database) +{ + ngx_http_geoip2_metadata_t *metadata; + ngx_str_t *value, name; + ngx_http_variable_t *var; + + metadata = ngx_pcalloc(cf->pool, sizeof(ngx_http_geoip2_metadata_t)); + if (metadata == NULL) { + return NGX_CONF_ERROR; + } + + value = cf->args->elts; + name = value[0]; + + metadata->database = database; + metadata->metavalue = value[2]; + + var = ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE); + if (var == NULL) { + return NGX_CONF_ERROR; + } + + var->get_handler = ngx_http_geoip2_metadata; + var->data = (uintptr_t) metadata; + + return NGX_CONF_OK; +} + + +static char * +ngx_http_geoip2_add_variable_geodata(ngx_conf_t *cf, ngx_http_geoip2_db_t *database) +{ + ngx_http_geoip2_ctx_t *geoip2; + ngx_http_compile_complex_value_t ccv; + ngx_str_t *value, name, source; + ngx_http_variable_t *var; + int i, nelts, idx; + + geoip2 = ngx_pcalloc(cf->pool, sizeof(ngx_http_geoip2_ctx_t)); + if (geoip2 == NULL) { + return NGX_CONF_ERROR; + } + + geoip2->database = database; + ngx_str_null(&source); + + value = cf->args->elts; + name = value[0]; + + nelts = (int) cf->args->nelts; + idx = 1; + + if (nelts > idx) { + for (i = idx; i < nelts; i++) { + if (ngx_strnstr(value[idx].data, "=", value[idx].len) == NULL) { + break; + } + + if (value[idx].len > 8 && ngx_strncmp(value[idx].data, "default=", 8) == 0) { + if (geoip2->default_value.len > 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "default has already been declared for \"$%V\"", &name); + return NGX_CONF_ERROR; + } + + geoip2->default_value.len = value[idx].len - 8; + geoip2->default_value.data = value[idx].data + 8; + } else if (value[idx].len > 7 && ngx_strncmp(value[idx].data, "source=", 7) == 0) { + if (source.len > 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "source has already been declared for \"$%V\"", &name); + return NGX_CONF_ERROR; + } + + source.len = value[idx].len - 7; + source.data = value[idx].data + 7; + + if (source.data[0] != '$') { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid source variable name \"%V\"", &source); + return NGX_CONF_ERROR; + } + + ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); + ccv.cf = cf; + ccv.value = &source; + ccv.complex_value = &geoip2->source; + + if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "unable to compile \"%V\" for \"$%V\"", &source, &name); + return NGX_CONF_ERROR; + } + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid setting \"%V\" for \"$%V\"", &value[idx], &name); + return NGX_CONF_ERROR; + } + + idx++; + } + } + + var = ngx_http_add_variable(cf, &name, NGX_HTTP_VAR_CHANGEABLE); + if (var == NULL) { + return NGX_CONF_ERROR; + } + + geoip2->lookup = ngx_pcalloc(cf->pool, sizeof(const char *) * + (cf->args->nelts - (idx - 1))); + + if (geoip2->lookup == NULL) { + return NGX_CONF_ERROR; + } + + for (i = idx; i < nelts; i++) { + geoip2->lookup[i - idx] = (char *) value[i].data; + } + geoip2->lookup[i - idx] = NULL; + + var->get_handler = ngx_http_geoip2_variable; + var->data = (uintptr_t) geoip2; + + return NGX_CONF_OK; +} + + +static char * +ngx_http_geoip2_init_conf(ngx_conf_t *cf, void *conf) +{ + ngx_http_geoip2_conf_t *gcf = conf; + ngx_conf_init_value(gcf->proxy_recursive, 0); + return NGX_CONF_OK; +} + + +static char * +ngx_http_geoip2_proxy(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_geoip2_conf_t *gcf = conf; + ngx_str_t *value; + ngx_cidr_t cidr, *c; + + value = cf->args->elts; + + if (ngx_http_geoip2_cidr_value(cf, &value[1], &cidr) != NGX_OK) { + return NGX_CONF_ERROR; + } + + if (gcf->proxies == NULL) { + gcf->proxies = ngx_array_create(cf->pool, 4, sizeof(ngx_cidr_t)); + if (gcf->proxies == NULL) { + return NGX_CONF_ERROR; + } + } + + c = ngx_array_push(gcf->proxies); + if (c == NULL) { + return NGX_CONF_ERROR; + } + + *c = cidr; + + return NGX_CONF_OK; +} + + +static ngx_int_t +ngx_http_geoip2_cidr_value(ngx_conf_t *cf, ngx_str_t *net, ngx_cidr_t *cidr) +{ + ngx_int_t rc; + + if (ngx_strcmp(net->data, "255.255.255.255") == 0) { + cidr->family = AF_INET; + cidr->u.in.addr = 0xffffffff; + cidr->u.in.mask = 0xffffffff; + + return NGX_OK; + } + + rc = ngx_ptocidr(net, cidr); + + if (rc == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid network \"%V\"", net); + return NGX_ERROR; + } + + if (rc == NGX_DONE) { + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "low address bits of %V are meaningless", net); + } + + return NGX_OK; +} + + +static void +ngx_http_geoip2_cleanup(void *data) +{ + ngx_http_geoip2_conf_t *gcf = data; + ngx_queue_t *q; + ngx_http_geoip2_db_t *database; + + while (!ngx_queue_empty(&gcf->databases)) { + q = ngx_queue_head(&gcf->databases); + ngx_queue_remove(q); + database = ngx_queue_data(q, ngx_http_geoip2_db_t, queue); + MMDB_close(&database->mmdb); + } +} + + +static ngx_int_t +ngx_http_geoip2_log_handler(ngx_http_request_t *r) +{ + int status; + MMDB_s tmpdb; + ngx_queue_t *q; + ngx_file_info_t fi; + ngx_http_geoip2_db_t *database; + ngx_http_geoip2_conf_t *gcf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "geoip2 http log handler"); + + gcf = ngx_http_get_module_main_conf(r, ngx_http_geoip2_module); + + if (ngx_queue_empty(&gcf->databases)) { + return NGX_OK; + } + + for (q = ngx_queue_head(&gcf->databases); + q != ngx_queue_sentinel(&gcf->databases); + q = ngx_queue_next(q)) + { + database = ngx_queue_data(q, ngx_http_geoip2_db_t, queue); + if (database->check_interval == 0) { + continue; + } + + if ((database->last_check + database->check_interval) + > ngx_time()) + { + continue; + } + + database->last_check = ngx_time(); + + if (ngx_file_info(database->mmdb.filename, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, r->connection->log, ngx_errno, + ngx_file_info_n " \"%s\" failed", + database->mmdb.filename); + + continue; + } + + if (ngx_file_mtime(&fi) <= database->last_change) { + continue; + } + + /* do the reload */ + + ngx_memzero(&tmpdb, sizeof(MMDB_s)); + status = MMDB_open(database->mmdb.filename, MMDB_MODE_MMAP, &tmpdb); + + if (status != MMDB_SUCCESS) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "MMDB_open(\"%s\") failed to reload - %s", + database->mmdb.filename, MMDB_strerror(status)); + + continue; + } + + database->last_change = ngx_file_mtime(&fi); + MMDB_close(&database->mmdb); + database->mmdb = tmpdb; + + ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, + "Reload MMDB \"%s\"", + database->mmdb.filename); + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_geoip2_init(ngx_conf_t *cf) +{ + ngx_http_handler_pt *h; + ngx_http_core_main_conf_t *cmcf; + + cmcf = ngx_http_conf_get_module_main_conf(cf, ngx_http_core_module); + + h = ngx_array_push(&cmcf->phases[NGX_HTTP_LOG_PHASE].handlers); + if (h == NULL) { + return NGX_ERROR; + } + + *h = ngx_http_geoip2_log_handler; + + return NGX_OK; +} diff --git a/images/proxy/ngx_http_geoip2_module/ngx_stream_geoip2_module.c b/images/proxy/ngx_http_geoip2_module/ngx_stream_geoip2_module.c new file mode 100644 index 000000000..eb590827f --- /dev/null +++ b/images/proxy/ngx_http_geoip2_module/ngx_stream_geoip2_module.c @@ -0,0 +1,694 @@ +/* + * Copyright (C) Lee Valentine + * Copyright (C) Andrei Belov + * + * Based on nginx's 'ngx_stream_geoip_module.c' by Igor Sysoev + */ + + +#include +#include +#include + +#include + + +typedef struct { + MMDB_s mmdb; + MMDB_lookup_result_s result; + time_t last_check; + time_t last_change; + time_t check_interval; +#if (NGX_HAVE_INET6) + uint8_t address[16]; +#else + unsigned long address; +#endif + ngx_queue_t queue; +} ngx_stream_geoip2_db_t; + +typedef struct { + ngx_queue_t databases; +} ngx_stream_geoip2_conf_t; + +typedef struct { + ngx_stream_geoip2_db_t *database; + const char **lookup; + ngx_str_t default_value; + ngx_stream_complex_value_t source; +} ngx_stream_geoip2_ctx_t; + +typedef struct { + ngx_stream_geoip2_db_t *database; + ngx_str_t metavalue; +} ngx_stream_geoip2_metadata_t; + + +static ngx_int_t ngx_stream_geoip2_variable(ngx_stream_session_t *s, + ngx_stream_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_stream_geoip2_metadata(ngx_stream_session_t *s, + ngx_stream_variable_value_t *v, uintptr_t data); +static void *ngx_stream_geoip2_create_conf(ngx_conf_t *cf); +static char *ngx_stream_geoip2(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_stream_geoip2_parse_config(ngx_conf_t *cf, ngx_command_t *dummy, + void *conf); +static char *ngx_stream_geoip2(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); +static char *ngx_stream_geoip2_add_variable(ngx_conf_t *cf, ngx_command_t *dummy, + void *conf); +static char *ngx_stream_geoip2_add_variable_geodata(ngx_conf_t *cf, + ngx_stream_geoip2_db_t *database); +static char *ngx_stream_geoip2_add_variable_metadata(ngx_conf_t *cf, + ngx_stream_geoip2_db_t *database); +static void ngx_stream_geoip2_cleanup(void *data); +static ngx_int_t ngx_stream_geoip2_init(ngx_conf_t *cf); + + +#define FORMAT(fmt, ...) do { \ + p = ngx_palloc(s->connection->pool, NGX_OFF_T_LEN); \ + if (p == NULL) { \ + return NGX_ERROR; \ + } \ + v->len = ngx_sprintf(p, fmt, __VA_ARGS__) - p; \ + v->data = p; \ +} while (0) + +static ngx_command_t ngx_stream_geoip2_commands[] = { + + { ngx_string("geoip2"), + NGX_STREAM_MAIN_CONF|NGX_CONF_BLOCK|NGX_CONF_TAKE1, + ngx_stream_geoip2, + NGX_STREAM_MAIN_CONF_OFFSET, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_stream_module_t ngx_stream_geoip2_module_ctx = { + NULL, /* preconfiguration */ + ngx_stream_geoip2_init, /* postconfiguration */ + + ngx_stream_geoip2_create_conf, /* create main configuration */ + NULL, /* init main configuration */ + + NULL, /* create server configuration */ + NULL /* merge server configuration */ +}; + + +ngx_module_t ngx_stream_geoip2_module = { + NGX_MODULE_V1, + &ngx_stream_geoip2_module_ctx, /* module context */ + ngx_stream_geoip2_commands, /* module directives */ + NGX_STREAM_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_stream_geoip2_variable(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, + uintptr_t data) +{ + int mmdb_error; + u_char *p; + ngx_str_t val; + ngx_addr_t addr; + MMDB_entry_data_s entry_data; + ngx_stream_geoip2_ctx_t *geoip2 = (ngx_stream_geoip2_ctx_t *) data; + ngx_stream_geoip2_db_t *database = geoip2->database; + +#if (NGX_HAVE_INET6) + uint8_t address[16], *addressp = address; +#else + unsigned long address; +#endif + + if (geoip2->source.value.len > 0) { + if (ngx_stream_complex_value(s, &geoip2->source, &val) != NGX_OK) { + goto not_found; + } + + if (ngx_parse_addr(s->connection->pool, &addr, val.data, val.len) != NGX_OK) { + goto not_found; + } + } else { + addr.sockaddr = s->connection->sockaddr; + addr.socklen = s->connection->socklen; + } + + switch (addr.sockaddr->sa_family) { + case AF_INET: +#if (NGX_HAVE_INET6) + ngx_memset(addressp, 0, 12); + ngx_memcpy(addressp + 12, &((struct sockaddr_in *) + addr.sockaddr)->sin_addr.s_addr, 4); + break; + + case AF_INET6: + ngx_memcpy(addressp, &((struct sockaddr_in6 *) + addr.sockaddr)->sin6_addr.s6_addr, 16); +#else + address = ((struct sockaddr_in *)addr.sockaddr)->sin_addr.s_addr; +#endif + break; + + default: + goto not_found; + } + +#if (NGX_HAVE_INET6) + if (ngx_memcmp(&address, &database->address, sizeof(address)) != 0) { +#else + if (address != database->address) { +#endif + memcpy(&database->address, &address, sizeof(address)); + database->result = MMDB_lookup_sockaddr(&database->mmdb, + addr.sockaddr, &mmdb_error); + + if (mmdb_error != MMDB_SUCCESS) { + goto not_found; + } + } + + if (!database->result.found_entry + || MMDB_aget_value(&database->result.entry, &entry_data, geoip2->lookup) + != MMDB_SUCCESS) + { + goto not_found; + } + + if (!entry_data.has_data) { + goto not_found; + } + + switch (entry_data.type) { + case MMDB_DATA_TYPE_BOOLEAN: + FORMAT("%d", entry_data.boolean); + break; + case MMDB_DATA_TYPE_UTF8_STRING: + v->len = entry_data.data_size; + v->data = ngx_pnalloc(s->connection->pool, v->len); + if (v->data == NULL) { + return NGX_ERROR; + } + ngx_memcpy(v->data, (u_char *) entry_data.utf8_string, v->len); + break; + case MMDB_DATA_TYPE_BYTES: + v->len = entry_data.data_size; + v->data = ngx_pnalloc(s->connection->pool, v->len); + if (v->data == NULL) { + return NGX_ERROR; + } + ngx_memcpy(v->data, (u_char *) entry_data.bytes, v->len); + break; + case MMDB_DATA_TYPE_FLOAT: + FORMAT("%.5f", entry_data.float_value); + break; + case MMDB_DATA_TYPE_DOUBLE: + FORMAT("%.5f", entry_data.double_value); + break; + case MMDB_DATA_TYPE_UINT16: + FORMAT("%uD", entry_data.uint16); + break; + case MMDB_DATA_TYPE_UINT32: + FORMAT("%uD", entry_data.uint32); + break; + case MMDB_DATA_TYPE_INT32: + FORMAT("%D", entry_data.int32); + break; + case MMDB_DATA_TYPE_UINT64: + FORMAT("%uL", entry_data.uint64); + break; + case MMDB_DATA_TYPE_UINT128: ; +#if MMDB_UINT128_IS_BYTE_ARRAY + uint8_t *val = (uint8_t *) entry_data.uint128; + FORMAT("0x%02x%02x%02x%02x%02x%02x%02x%02x" + "%02x%02x%02x%02x%02x%02x%02x%02x", + val[0], val[1], val[2], val[3], + val[4], val[5], val[6], val[7], + val[8], val[9], val[10], val[11], + val[12], val[13], val[14], val[15]); +#else + mmdb_uint128_t val = entry_data.uint128; + FORMAT("0x%016uxL%016uxL", + (uint64_t) (val >> 64), (uint64_t) val); +#endif + break; + default: + goto not_found; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; + +not_found: + if (geoip2->default_value.len > 0) { + v->data = geoip2->default_value.data; + v->len = geoip2->default_value.len; + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; + } + + v->not_found = 1; + + return NGX_OK; +} + + +static ngx_int_t +ngx_stream_geoip2_metadata(ngx_stream_session_t *s, ngx_stream_variable_value_t *v, + uintptr_t data) +{ + ngx_stream_geoip2_metadata_t *metadata = (ngx_stream_geoip2_metadata_t *) data; + ngx_stream_geoip2_db_t *database = metadata->database; + u_char *p; + + if (ngx_strncmp(metadata->metavalue.data, "build_epoch", 11) == 0) { + FORMAT("%uL", database->mmdb.metadata.build_epoch); + } else if (ngx_strncmp(metadata->metavalue.data, "last_check", 10) == 0) { + FORMAT("%T", database->last_check); + } else if (ngx_strncmp(metadata->metavalue.data, "last_change", 11) == 0) { + FORMAT("%T", database->last_change); + } else { + v->not_found = 1; + return NGX_OK; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + return NGX_OK; +} + + +static void * +ngx_stream_geoip2_create_conf(ngx_conf_t *cf) +{ + ngx_pool_cleanup_t *cln; + ngx_stream_geoip2_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, sizeof(ngx_stream_geoip2_conf_t)); + if (conf == NULL) { + return NULL; + } + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NULL; + } + + ngx_queue_init(&conf->databases); + + cln->handler = ngx_stream_geoip2_cleanup; + cln->data = conf; + + return conf; +} + + +static char * +ngx_stream_geoip2(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + int status; + char *rv; + ngx_str_t *value; + ngx_conf_t save; + ngx_stream_geoip2_db_t *database; + ngx_stream_geoip2_conf_t *gcf = conf; + ngx_queue_t *q; + + value = cf->args->elts; + + if (value[1].data && value[1].data[0] != '/') { + if (ngx_conf_full_name(cf->cycle, &value[1], 0) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + + if (!ngx_queue_empty(&gcf->databases)) { + for (q = ngx_queue_head(&gcf->databases); + q != ngx_queue_sentinel(&gcf->databases); + q = ngx_queue_next(q)) + { + database = ngx_queue_data(q, ngx_stream_geoip2_db_t, queue); + if (ngx_strcmp(value[1].data, database->mmdb.filename) == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "Duplicate GeoIP2 mmdb - %V", &value[1]); + return NGX_CONF_ERROR; + } + } + } + + database = ngx_pcalloc(cf->pool, sizeof(ngx_stream_geoip2_db_t)); + if (database == NULL) { + return NGX_CONF_ERROR; + } + + ngx_queue_insert_tail(&gcf->databases, &database->queue); + database->last_check = database->last_change = ngx_time(); + + status = MMDB_open((char *) value[1].data, MMDB_MODE_MMAP, &database->mmdb); + + if (status != MMDB_SUCCESS) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "MMDB_open(\"%V\") failed - %s", &value[1], + MMDB_strerror(status)); + return NGX_CONF_ERROR; + } + + save = *cf; + cf->handler = ngx_stream_geoip2_parse_config; + cf->handler_conf = (void *) database; + + rv = ngx_conf_parse(cf, NULL); + *cf = save; + return rv; +} + + +static char * +ngx_stream_geoip2_parse_config(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) +{ + ngx_stream_geoip2_db_t *database; + ngx_str_t *value; + time_t interval; + + value = cf->args->elts; + + if (value[0].data[0] == '$') { + return ngx_stream_geoip2_add_variable(cf, dummy, conf); + } + + if (value[0].len == 11 + && ngx_strncmp(value[0].data, "auto_reload", 11) == 0) { + if ((int) cf->args->nelts != 2) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid number of arguments for auto_reload"); + return NGX_CONF_ERROR; + } + + interval = ngx_parse_time(&value[1], true); + + if (interval == (time_t) NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid interval for auto_reload \"%V\"", + value[1]); + return NGX_CONF_ERROR; + } + + + database = (ngx_stream_geoip2_db_t *) conf; + database->check_interval = interval; + return NGX_CONF_OK; + } + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid setting \"%V\"", &value[0]); + return NGX_CONF_ERROR; +} + + +static char * +ngx_stream_geoip2_add_variable(ngx_conf_t *cf, ngx_command_t *dummy, void *conf) +{ + ngx_stream_geoip2_db_t *database; + ngx_str_t *value; + int nelts; + + value = cf->args->elts; + + if (value[0].data[0] != '$') { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid variable name \"%V\"", &value[0]); + return NGX_CONF_ERROR; + } + + value[0].len--; + value[0].data++; + + nelts = (int) cf->args->nelts; + database = (ngx_stream_geoip2_db_t *) conf; + + if (nelts > 0 && value[1].len == 8 && ngx_strncmp(value[1].data, "metadata", 8) == 0) { + return ngx_stream_geoip2_add_variable_metadata(cf, database); + } + + return ngx_stream_geoip2_add_variable_geodata(cf, database); +} + + +static char * +ngx_stream_geoip2_add_variable_metadata(ngx_conf_t *cf, ngx_stream_geoip2_db_t *database) +{ + ngx_stream_geoip2_metadata_t *metadata; + ngx_str_t *value, name; + ngx_stream_variable_t *var; + + metadata = ngx_pcalloc(cf->pool, sizeof(ngx_stream_geoip2_metadata_t)); + if (metadata == NULL) { + return NGX_CONF_ERROR; + } + + value = cf->args->elts; + name = value[0]; + + metadata->database = database; + metadata->metavalue = value[2]; + + var = ngx_stream_add_variable(cf, &name, NGX_STREAM_VAR_CHANGEABLE); + if (var == NULL) { + return NGX_CONF_ERROR; + } + + var->get_handler = ngx_stream_geoip2_metadata; + var->data = (uintptr_t) metadata; + + return NGX_CONF_OK; +} + + +static char * +ngx_stream_geoip2_add_variable_geodata(ngx_conf_t *cf, ngx_stream_geoip2_db_t *database) +{ + ngx_stream_geoip2_ctx_t *geoip2; + ngx_stream_compile_complex_value_t ccv; + ngx_str_t *value, name, source; + ngx_stream_variable_t *var; + int i, nelts, idx; + + geoip2 = ngx_pcalloc(cf->pool, sizeof(ngx_stream_geoip2_ctx_t)); + if (geoip2 == NULL) { + return NGX_CONF_ERROR; + } + + geoip2->database = database; + ngx_str_null(&source); + + value = cf->args->elts; + name = value[0]; + + nelts = (int) cf->args->nelts; + idx = 1; + + if (nelts > idx) { + for (i = idx; i < nelts; i++) { + if (ngx_strnstr(value[idx].data, "=", value[idx].len) == NULL) { + break; + } + + if (value[idx].len > 8 && ngx_strncmp(value[idx].data, "default=", 8) == 0) { + if (geoip2->default_value.len > 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "default has already been declared for \"$%V\"", &name); + return NGX_CONF_ERROR; + } + + geoip2->default_value.len = value[idx].len - 8; + geoip2->default_value.data = value[idx].data + 8; + + } else if (value[idx].len > 7 && ngx_strncmp(value[idx].data, "source=", 7) == 0) { + if (source.len > 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "source has already been declared for \"$%V\"", &name); + return NGX_CONF_ERROR; + } + + source.len = value[idx].len - 7; + source.data = value[idx].data + 7; + + if (source.data[0] != '$') { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid source variable name \"%V\"", &source); + return NGX_CONF_ERROR; + } + + ngx_memzero(&ccv, sizeof(ngx_stream_compile_complex_value_t)); + ccv.cf = cf; + ccv.value = &source; + ccv.complex_value = &geoip2->source; + + if (ngx_stream_compile_complex_value(&ccv) != NGX_OK) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "unable to compile \"%V\" for \"$%V\"", &source, &name); + return NGX_CONF_ERROR; + } + + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid setting \"%V\" for \"$%V\"", &value[idx], &name); + return NGX_CONF_ERROR; + } + + idx++; + } + } + + var = ngx_stream_add_variable(cf, &name, NGX_STREAM_VAR_CHANGEABLE); + if (var == NULL) { + return NGX_CONF_ERROR; + } + + geoip2->lookup = ngx_pcalloc(cf->pool, + sizeof(const char *) * (cf->args->nelts - (idx - 1))); + + if (geoip2->lookup == NULL) { + return NGX_CONF_ERROR; + } + + for (i = idx; i < nelts; i++) { + geoip2->lookup[i - idx] = (char *) value[i].data; + } + geoip2->lookup[i - idx] = NULL; + + var->get_handler = ngx_stream_geoip2_variable; + var->data = (uintptr_t) geoip2; + + return NGX_CONF_OK; +} + + +static void +ngx_stream_geoip2_cleanup(void *data) +{ + ngx_queue_t *q; + ngx_stream_geoip2_db_t *database; + ngx_stream_geoip2_conf_t *gcf = data; + + while (!ngx_queue_empty(&gcf->databases)) { + q = ngx_queue_head(&gcf->databases); + ngx_queue_remove(q); + database = ngx_queue_data(q, ngx_stream_geoip2_db_t, queue); + MMDB_close(&database->mmdb); + } +} + + +static ngx_int_t +ngx_stream_geoip2_log_handler(ngx_stream_session_t *s) +{ + int status; + MMDB_s tmpdb; + ngx_queue_t *q; + ngx_file_info_t fi; + ngx_stream_geoip2_db_t *database; + ngx_stream_geoip2_conf_t *gcf; + + ngx_log_debug0(NGX_LOG_DEBUG_STREAM, s->connection->log, 0, + "geoip2 stream log handler"); + + gcf = ngx_stream_get_module_main_conf(s, ngx_stream_geoip2_module); + + if (ngx_queue_empty(&gcf->databases)) { + return NGX_OK; + } + + for (q = ngx_queue_head(&gcf->databases); + q != ngx_queue_sentinel(&gcf->databases); + q = ngx_queue_next(q)) + { + database = ngx_queue_data(q, ngx_stream_geoip2_db_t, queue); + if (database->check_interval == 0) { + continue; + } + + if ((database->last_check + database->check_interval) + > ngx_time()) + { + continue; + } + + database->last_check = ngx_time(); + + if (ngx_file_info(database->mmdb.filename, &fi) == NGX_FILE_ERROR) { + ngx_log_error(NGX_LOG_EMERG, s->connection->log, ngx_errno, + ngx_file_info_n " \"%s\" failed", + database->mmdb.filename); + + continue; + } + + if (ngx_file_mtime(&fi) <= database->last_change) { + continue; + } + + /* do the reload */ + + ngx_memzero(&tmpdb, sizeof(MMDB_s)); + status = MMDB_open(database->mmdb.filename, MMDB_MODE_MMAP, &tmpdb); + + if (status != MMDB_SUCCESS) { + ngx_log_error(NGX_LOG_ERR, s->connection->log, 0, + "MMDB_open(\"%s\") failed to reload - %s", + database->mmdb.filename, MMDB_strerror(status)); + + continue; + } + + database->last_change = ngx_file_mtime(&fi); + MMDB_close(&database->mmdb); + database->mmdb = tmpdb; + + ngx_log_error(NGX_LOG_INFO, s->connection->log, 0, + "Reload MMDB \"%s\"", + database->mmdb.filename); + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_stream_geoip2_init(ngx_conf_t *cf) +{ + ngx_stream_handler_pt *h; + ngx_stream_core_main_conf_t *cmcf; + + cmcf = ngx_stream_conf_get_module_main_conf(cf, ngx_stream_core_module); + + h = ngx_array_push(&cmcf->phases[NGX_STREAM_LOG_PHASE].handlers); + if (h == NULL) { + return NGX_ERROR; + } + + *h = ngx_stream_geoip2_log_handler; + + return NGX_OK; +} diff --git a/kubernetes/ansible/api-manager.yml b/kubernetes/ansible/api-manager.yml new file mode 100644 index 000000000..77562241f --- /dev/null +++ b/kubernetes/ansible/api-manager.yml @@ -0,0 +1,11 @@ +--- +- name: Kong API + hosts: local + become: yes + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + vars_files: + - "{{inventory_dir}}/secrets.yml" + roles: + - {role: kong-api, tags: ['kong-api']} + - {role: kong-consumer, tags: ['kong-consumer']} diff --git a/kubernetes/ansible/bootstrap.yaml b/kubernetes/ansible/bootstrap.yaml new file mode 100644 index 000000000..02cafaeb6 --- /dev/null +++ b/kubernetes/ansible/bootstrap.yaml @@ -0,0 +1,46 @@ +--- +# tasks file for bootstrap-k8s + +- hosts: local + gather_facts: false + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + serial: 1 + vars: + # This is where any custom config for helm_charts to define + # this file will get templatized in /tmp/helm_vars.yaml + # and use it in helm upgrade --install .. -f /tmp/helm_vars.yaml + # example: + # chartname: + # variables + # .... + helm_vars: + reloader: + watchGlobally: false + deployment: + pod: + annotations: + sidecar.istio.io/inject: "false" + vars_files: + - "{{inventory_dir}}/secrets.yml" + pre_tasks: + - name: Creating helm_charts vars file + copy: + content: "{{ helm_vars|to_yaml }}" + dest: /tmp/helm_vars.yaml + - name: Creating namespace + shell: "kubectl create namespace {{ item }} " + with_items: + - "{{ namespace }}" + - monitoring + - logging + - istio-system + ignore_errors: yes + - name: "Tagging {{ namespace }} to enable istio" + shell: "kubectl label namespaces {{ namespace }} istio-injection=enabled --overwrite" + - name: Creating docker secrets + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run -o=yaml | kubectl apply -f -" + - name: Installing reloader for configmaps reload + shell: helm upgrade --install --atomic reloader ../helm_charts/core/reloader --namespace "{{ namespace }}" -f /tmp/helm_vars.yaml + roles: + - istio diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml new file mode 100644 index 000000000..1a6420dcd --- /dev/null +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -0,0 +1,67 @@ +--- +# tasks file for bootstrap-k8s + +- hosts: local + gather_facts: false + environment: + KUBECONFIG: "{{ kubeconfig_path }}" + serial: 1 + vars: + helm_vars: + reloader: + watchGlobally: false + vars_files: + - "{{inventory_dir}}/secrets.yml" + pre_tasks: + - name: Creating tmp file + tempfile: + state: file + suffix: helm + register: helm_file + - name: Creating helm_charts vars file + copy: + content: "{{ helm_vars|to_yaml }}" + dest: "{{ helm_file.path }}" + - name: Creating namespace + shell: "kubectl create namespace {{ item }} " + with_items: + - "{{ namespace }}" + - monitoring + - logging + ignore_errors: yes + - name: Creating docker secrets + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run -o=yaml | kubectl apply -f -" + when: imagepullsecrets|length > 0 + - name: Creating domain ssl + shell: + cmd: | + cat < /dev/null diff --git a/kubernetes/ansible/roles/deploy-player/defaults/main.yml b/kubernetes/ansible/roles/deploy-player/defaults/main.yml new file mode 120000 index 000000000..b6acc0109 --- /dev/null +++ b/kubernetes/ansible/roles/deploy-player/defaults/main.yml @@ -0,0 +1 @@ +../../../../../ansible/roles/stack-sunbird/defaults/main.yml \ No newline at end of file diff --git a/kubernetes/ansible/roles/deploy-player/tasks/main.yml b/kubernetes/ansible/roles/deploy-player/tasks/main.yml new file mode 100644 index 000000000..25fb73c77 --- /dev/null +++ b/kubernetes/ansible/roles/deploy-player/tasks/main.yml @@ -0,0 +1,85 @@ +- name: remove blank lines in index file + shell: "awk 'NF' {{cdn_file_path}} > /tmp/player-cdn-config" + when: sunbird_portal_player_cdn_enabled == 'true' + +- name: remove trailing spaces in index file + shell: "awk '{$1=$1;print}' /tmp/player-cdn-config > {{cdn_file_path}} && rm /tmp/player-cdn-config" + when: sunbird_portal_player_cdn_enabled == 'true' + +- name: Remove the existing CDN config + shell: "kubectl delete configmap player-cdn-config --namespace={{namespace}}" + ignore_errors: true + +- name: Create player-cdn configmap + shell: "kubectl create configmap player-cdn-config --from-file={{cdn_file_path}} --namespace={{namespace}}" + when: sunbird_portal_player_cdn_enabled == 'true' + +- name: rename template + template: + src: "{{ chart_path }}/values.j2" + dest: "{{ chart_path }}/values.yaml" + +- name: copy env file from swarm role + template: + src: "{{service_env[release_name]}}" + dest: "{{role_path}}/templates/{{ release_name }}.env" + +- name: template vars + template: + src: "{{ release_name }}.env" + dest: "{{role_path}}/templates/{{ release_name }}.env" + +- name: create configmap + shell: "kubectl create configmap player-config --from-env-file={{role_path}}/templates/{{ release_name }}.env -n {{namespace}} --dry-run -o=yaml | kubectl apply -f -" + +- name: Load role to decrypt private keys, copy to helm chart and encrypt private keys + include_role: + name: mount-keys + tasks_from: "{{outer_item.1}}" + vars: + private_key_path: "{{ outer_item.0.values_to_pass.basepath }}" + private_key_prefix: "{{ outer_item.0.values_to_pass.keyprefix }}" + private_key_sign_start: "{{ outer_item.0.values_to_pass.keystart }}" + private_key_sign_end: "{{ outer_item.0.values_to_pass.keycount if outer_item.0.values_to_pass.keycount > '0' else '1' }}" + when: release_name == "player" + with_subelements: + - "{{adminutil_access_values}}" + - role_to_run + loop_control: + loop_var: outer_item + +- name: Create the token pubic key file + copy: + dest: "{{ chart_path }}/keys/{{ adminutil_refresh_token_public_key_kid }}" + content: "{{ core_vault_sunbird_sso_publickey }}" + when: release_name == "player" + +- name: helm install and upgrade + shell: helm upgrade --install --atomic --timeout 10m {{ release_name }} {{ chart_path }} -n {{namespace}} + +- name: Clean up keys + include_role: + name: mount-keys + tasks_from: "{{item}}" + when: release_name == "player" + with_items: + - remove-from-helm.yml + - remove-keys-from-inventory.yml + +- name: Get the deployment rollout status + shell: "kubectl get deployments -A | grep -i {{ release_name }} | awk -F' ' '{print $3}' | awk -F/ '{if ($1 ~ $2){exit 0} else {exit 1}}'" + register: task_result + until: task_result.rc == 0 + retries: 6 + delay: 30 + +- name: Get deployed image name + shell: kubectl get deployments.apps {{ release_name }} -o json -n {{ namespace }} | jq -r '.spec.template.spec.containers[0].image | split("/")[1]' + register: deployed_image + +- set_fact: + metadata_image: "{{ image_name }}:{{ image_tag }}" + +- name: Compare deployed image and metadata_image + fail: + when: deployed_image.stdout != metadata_image diff --git a/kubernetes/ansible/roles/deploy-player/templates/service.env b/kubernetes/ansible/roles/deploy-player/templates/service.env new file mode 100644 index 000000000..e69de29bb diff --git a/kubernetes/ansible/roles/helm-daemonset/defaults/main.yml b/kubernetes/ansible/roles/helm-daemonset/defaults/main.yml new file mode 100644 index 000000000..59eb13677 --- /dev/null +++ b/kubernetes/ansible/roles/helm-daemonset/defaults/main.yml @@ -0,0 +1,38 @@ +--- +hub_org: sunbird +nginx_per_ip_connection_limit: 400 +merge_proxy_server_name: +merge_domain_status: false +proxy_prometheus: false + +proxy_secrets: + - name: prom_admin_creds + value: "{{proxy_prometheus_admin_creds}}" + - name: site.key + value: "{{proxy_site_key}}" + - name: site.crt + value: "{{proxy_site_crt}}" + +merge_proxy_secrets: + - name: sitemerge.key + value: "{{proxymerge_site_key}}" + - name: sitemerge.crt + value: "{{proxymerge_site_crt}}" + +proxy_custom_config: + +# These values should be overridden in inventory +proxy_site_key: +proxy_site_crt: +proxymerge_site_key: +proxymerge_site_crt: +proxy_prometheus_admin_creds: +prometheus_route_prefix: prometheus +prometheus_alertmanager_route_prefix: alertmanager +ekstep_s3_env: "{{sunbird_env}}" +registry_url: "{{proto}}://{{proxy_server_name}}/registry" +ep_es_host: + +upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +plugin_upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +sunbird_offline_azure_storage_account_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ offline_installer_container_name }}" diff --git a/kubernetes/ansible/roles/helm-daemonset/tasks/main.yml b/kubernetes/ansible/roles/helm-daemonset/tasks/main.yml new file mode 100644 index 000000000..cfa188ddb --- /dev/null +++ b/kubernetes/ansible/roles/helm-daemonset/tasks/main.yml @@ -0,0 +1,36 @@ +# This variable will get injected to nginx resolver config +# There is no native helm way to do this. So it's +# kind of a workaround +- name: getting kube dns ip + shell: kubectl get service --namespace kube-system kube-dns -o jsonpath="{.spec.clusterIP}" + register: kube_dns_ip + +- name: setting variable + set_fact: + kube_dns_ip: "{{ kube_dns_ip.stdout }}" + +- name: rename template + template: + src: "{{ chart_path }}/values.j2" + dest: "{{ chart_path }}/values.yaml" + +- name: helm install + shell: helm upgrade --install --atomic --timeout 10m {{ release_name }} {{ chart_path }} -n {{namespace}} + +- name: Get the deployment rollout status + shell: "kubectl get daemonsets -A | grep -i {{ release_name }} | awk -F' ' '{if ($3 ~ $6){exit 0} else {exit 1}}'" + register: task_result + until: task_result.rc == 0 + retries: 6 + delay: 30 + +- name: Get deployed image name + shell: "kubectl get daemonsets {{ release_name }} -o json -n {{ namespace }} | jq '.spec.template.spec.containers | .[].image' -r | awk -F/ '{print $2}'" + register: deployed_image + +- set_fact: + metadata_image: "{{ image_name }}:{{ image_tag }}" + +- name: Compare deployed image and metadata_image + fail: + when: deployed_image.stdout != metadata_image diff --git a/kubernetes/ansible/roles/helm-deploy/defaults/main.yml b/kubernetes/ansible/roles/helm-deploy/defaults/main.yml new file mode 100644 index 000000000..6e960fbb8 --- /dev/null +++ b/kubernetes/ansible/roles/helm-deploy/defaults/main.yml @@ -0,0 +1,122 @@ +--- +hub_org: sunbird +nginx_per_ip_connection_limit: 400 +merge_proxy_server_name: +merge_domain_status: false +proxy_prometheus: false + +proxy_secrets: + - name: prom_admin_creds + value: "{{proxy_prometheus_admin_creds}}" + - name: site.key + value: "{{proxy_site_key}}" + - name: site.crt + value: "{{proxy_site_crt}}" + +merge_proxy_secrets: + - name: sitemerge.key + value: "{{proxymerge_site_key}}" + - name: sitemerge.crt + value: "{{proxymerge_site_crt}}" + +# These values should be overridden in inventory +proxy_site_key: +proxy_site_crt: +proxymerge_site_key: +proxymerge_site_crt: +proxy_prometheus_admin_creds: +prometheus_route_prefix: prometheus +prometheus_alertmanager_route_prefix: alertmanager +ekstep_s3_env: "{{sunbird_env}}" +registry_url: "{{proto}}://{{proxy_server_name}}/registry" +ep_es_host: +proxy_custom_config: +kibana_service: "{{swarm_dashboard}}:5601" + +upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +plugin_upstream_url: "ekstep-public-{{ekstep_s3_env}}.s3-ap-south-1.amazonaws.com" +sunbird_offline_azure_storage_account_url: "{{ sunbird_public_storage_account_name }}.blob.core.windows.net/{{ offline_installer_container_name }}" +adminutils_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + successThreshold: 2 + livenessProbe: + httpGet: + path: /health + port: 4000 + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 5 + +api_managerecho_liveness_readiness: + healthcheck: true + readinessProbe: + httpGet: + path: /hello + port: 9595 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + livenessProbe: + httpGet: + path: /hello + port: 9595 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 5 + +adminutil__device_keyprefix: "device" +adminutil__access_keyprefix: "access" +adminutil__desktop_keyprefix: "desktop" +adminutil__device_keystart: 0 +adminutil__device_keycount: 0 +adminutil__access_keystart: 0 +adminutil__access_keycount: 0 +adminutil__desktop_keystart: 0 +adminutil__desktop_keycount: 0 + +adminutil_refresh_token_public_key_prefix: "refresh_token_public_key" +adminutil_access_token_validity: 43200 # 12 hours access token validity +adminutil_refresh_token_offline_validity: 15552000 # 180 days offline token validity +adminutil_refresh_token_log_older_than: 30 # Write a log when the token is older than 30 days +adminutil_refresh_token_preload: "true" + +adminutil_keys_values: +- key_to_register: device + values_to_pass: + basepath: "/keys/" + keyprefix: "{{ adminutil__device_keyprefix }}" + keystart: "{{ adminutil__device_keystart }}" + keycount: "{{ adminutil__device_keycount|int }}" + role_to_run: + - decrypt.yml + - copy-to-helm.yml +- key_to_register: access + values_to_pass: + basepath: "/keys/" + keyprefix: "{{ adminutil__access_keyprefix }}" + keystart: "{{ adminutil__access_keystart }}" + keycount: "{{ adminutil__access_keycount|int }}" + role_to_run: + - decrypt.yml + - copy-to-helm.yml +- key_to_register: desktop + values_to_pass: + basepath: "/keys/" + keyprefix: "{{ adminutil__desktop_keyprefix }}" + keystart: "{{ adminutil__desktop_keystart }}" + keycount: "{{ adminutil__desktop_keycount|int }}" + role_to_run: + - decrypt.yml + - copy-to-helm.yml diff --git a/kubernetes/ansible/roles/helm-deploy/tasks/main.yml b/kubernetes/ansible/roles/helm-deploy/tasks/main.yml new file mode 100644 index 000000000..9ef163db2 --- /dev/null +++ b/kubernetes/ansible/roles/helm-deploy/tasks/main.yml @@ -0,0 +1,107 @@ +# This variable will get injected to nginx resolver config +# There is no native helm way to do this. So it's +# kind of a workaround +- name: getting kube dns ip + shell: kubectl get service --namespace kube-system kube-dns -o jsonpath="{.spec.clusterIP}" + register: kube_dns_ip + +- name: setting variable + set_fact: + kube_dns_ip: "{{ kube_dns_ip.stdout }}" + +- name: rename template + template: + src: "{{ chart_path }}/values.j2" + dest: "{{ chart_path }}/values.yaml" + +- name: Load role to decrypt private keys, copy to helm chart and encrypt private keys + include_role: + name: mount-keys + tasks_from: "{{outer_item.1}}" + vars: + private_key_path: "{{ outer_item.0.values_to_pass.basepath }}" + private_key_prefix: "{{ outer_item.0.values_to_pass.keyprefix }}" + private_key_sign_start: "{{ outer_item.0.values_to_pass.keystart }}" + private_key_sign_end: "{{ outer_item.0.values_to_pass.keycount if outer_item.0.values_to_pass.keycount > '0' else '1' }}" + when: release_name == "adminutils" + with_subelements: + - "{{adminutil_keys_values}}" + - role_to_run + loop_control: + loop_var: outer_item + +- name: Create the token pubic key file + copy: + dest: "{{ chart_path }}/keys/{{ adminutil_refresh_token_public_key_prefix }}" + content: "{{ core_vault_sunbird_sso_publickey }}" + when: release_name == "adminutils" + +- name: helm install + shell: helm upgrade --install --atomic --timeout 10m {{ release_name }} {{ chart_path }} -n {{namespace}} + +- name: Clean up keys + include_role: + name: mount-keys + tasks_from: "{{item}}" + when: release_name == "adminutils" + with_items: + - remove-from-helm.yml + - remove-keys-from-inventory.yml + +- name: Get the deployment rollout status + shell: "kubectl get deployments -A | grep -i {{ release_name }} | if [[ $(wc -l) > 0 ]]; then awk '{print $3}' | awk -F/ '{if ($1 ~ $2){exit 0} else {exit 1}}'; else exit 1; fi" + args: + executable: /bin/bash + register: deployment_result + ignore_errors: yes + +- name: Get deployed image name - deployments + shell: "kubectl get deployments {{ release_name }} -o json -n {{ namespace }} | jq '.spec.template.spec.containers | .[].image' -r | awk -F/ '{print $2}'" + register: image + +- set_fact: + deployed_image: "{{ image }}" + +- name: Get the daemonset rollout status + shell: "kubectl get daemonsets -A | grep -i {{ release_name }} | if [[ $(wc -l) > 0 ]]; then awk '{print $3}' | awk -F/ '{if ($1 ~ $2){exit 0} else {exit 1}}'; else exit 1; fi" + args: + executable: /bin/bash + register: daemonset_result + ignore_errors: yes + +- name: Get deployed image name - daemonsets + shell: "kubectl get daemonsets {{ release_name }} -o json -n {{ namespace }} | jq '.spec.template.spec.containers | .[].image' -r | awk -F/ '{print $2}'" + register: image + +- set_fact: + deployed_image: "{{ image if image.stdout_lines | length > 0 else deployed_image }}" + +- name: Get the statefulset rollout status + shell: "kubectl get statefulsets -A | grep -i {{ release_name }} | if [[ $(wc -l) > 0 ]]; then awk '{print $3}' | awk -F/ '{if ($1 ~ $2){exit 0} else {exit 1}}'; else exit 1; fi" + args: + executable: /bin/bash + register: statefulset_result + ignore_errors: yes + +- name: Get deployed image name - statefulsets + shell: "kubectl get statefulsets {{ release_name }} -o json -n {{ namespace }} | jq '.spec.template.spec.containers | .[].image' -r | awk -F/ '{print $2}'" + register: image + +- set_fact: + deployed_image: "{{ image if image.stdout_lines | length > 0 else deployed_image }}" + +- name: fail when deployment or daemonset or statefulset are not successful + fail: + msg: "Unable to get success status for any of these - deployment, daemonset, statefulset" + when: statefulset_result.rc != 0 and daemonset_result.rc != 0 and deployment_result.rc != 0 + +- set_fact: + metadata_image: "{{ image_name }}:{{ image_tag }}" + +- debug: + msg: "deployed image: {{ deployed_image.stdout }}" + +- name: Compare deployed image and metadata_image + fail: + msg: "Image version from Jenkins job and actual deployed image version do not match" + when: deployed_image.stdout != metadata_image diff --git a/kubernetes/ansible/roles/istio/README.md b/kubernetes/ansible/roles/istio/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/kubernetes/ansible/roles/istio/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/kubernetes/ansible/roles/istio/defaults/main.yml b/kubernetes/ansible/roles/istio/defaults/main.yml new file mode 100644 index 000000000..908ffab1e --- /dev/null +++ b/kubernetes/ansible/roles/istio/defaults/main.yml @@ -0,0 +1,4 @@ +--- +# defaults file for istio + + diff --git a/kubernetes/ansible/roles/istio/handlers/main.yml b/kubernetes/ansible/roles/istio/handlers/main.yml new file mode 100644 index 000000000..01778cdfb --- /dev/null +++ b/kubernetes/ansible/roles/istio/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for istio \ No newline at end of file diff --git a/kubernetes/ansible/roles/istio/meta/main.yml b/kubernetes/ansible/roles/istio/meta/main.yml new file mode 100644 index 000000000..722379974 --- /dev/null +++ b/kubernetes/ansible/roles/istio/meta/main.yml @@ -0,0 +1,57 @@ +galaxy_info: + author: your name + description: your description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 1.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. \ No newline at end of file diff --git a/kubernetes/ansible/roles/istio/tasks/main.yml b/kubernetes/ansible/roles/istio/tasks/main.yml new file mode 100644 index 000000000..4b89486d8 --- /dev/null +++ b/kubernetes/ansible/roles/istio/tasks/main.yml @@ -0,0 +1,35 @@ +--- +# tasks file for istio +- name: tempating variables + template: + src: "{{ item }}.yaml" + dest: "/tmp/{{item}}.yaml" + with_items: + - istio + +- name: initializing istio-crds + shell: helm upgrade --install --atomic istio-init {{ chart_path }}/istio-init --namespace istio-system + +- name: waiting for crds to get completed + pause: + seconds: 30 + +- name: Creating kiali secrtes + shell: + cmd: | + cat < + @type relabel + @label @NORMAL + + + + containers.input.conf: |- + + @id fluentd-containers.log + @type tail + path /var/log/containers/*.log + pos_file /var/log/containers.log.pos + tag raw.kubernetes.* + read_from_head true + + @type multi_format + + format json + time_key time + time_format %Y-%m-%dT%H:%M:%S.%NZ + + + format /^(? + + + + # Detect exceptions in the log output and forward them as one log entry. + + @id raw.kubernetes + @type detect_exceptions + remove_tag_prefix raw + message log + stream stream + multiline_flush_interval 5 + max_bytes 500000 + max_lines 1000 + + + # Concatenate multi-line logs + + @id filter_concat + @type concat + key message + multiline_end_regexp /\n$/ + separator "" + timeout_label @NORMAL + flush_interval 5 + + + # Enriches records with Kubernetes metadata + + @id filter_kubernetes_metadata + @type kubernetes_metadata + + + # Fixes json fields in Elasticsearch + + @id filter_parser + @type parser + key_name log + reserve_time true + reserve_data true + remove_key_name_field true + + @type multi_format + + format json + + + format none + + + + + # To add filtering capability of the proxy logs + + @type parser + key_name message + reserve_time true + reserve_data true + remove_key_name_field false + + @type regexp + expression ^(?[^ ]*) (?[^ ]*) (?[^ ]*) \[(? + + + # To remove the unwanted fields + + @type record_transformer + remove_keys uid,host,referer,agent,pipe,verb + + + # to exclude the lines which has specific pattern + + @type grep + + key message + pattern /(Completed checkpoint|Triggering checkpoint)/ + + diff --git a/kubernetes/ansible/roles/logging/templates/kibana.yaml b/kubernetes/ansible/roles/logging/templates/kibana.yaml new file mode 100644 index 000000000..31063dedc --- /dev/null +++ b/kubernetes/ansible/roles/logging/templates/kibana.yaml @@ -0,0 +1,37 @@ +replicas: {{ kibana_replicas | default(1) }} +image: "docker.elastic.co/kibana/kibana" +imageTag: "6.8.6" +imagePullPolicy: "IfNotPresent" + +resources: + requests: + cpu: "100m" + memory: "100Mi" + limits: + cpu: "4" + memory: "4Gi" + +healthCheckPath: "/dashboard" + +kibanaConfig: + kibana.yml: | + server.basePath: "/dashboard" + server.host: "0.0.0.0" + server.rewriteBasePath: true + elasticsearch.shardTimeout: 0 + elasticsearch.hosts: ["http://{{groups['log-es'] | join(':9200","http://')}}:9200"] + elasticsearch.requestTimeout: 300000 + elasticsearch.preserveHost: true + elasticsearch.ssl.verificationMode: full + +updateStrategy: + type: "RollingUpdate" + +readinessProbe: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + failureThreshold: 10 + successThreshold: 1 + +fullnameOverride: "kibana" diff --git a/kubernetes/ansible/roles/logging/templates/oauth2-proxy.yaml b/kubernetes/ansible/roles/logging/templates/oauth2-proxy.yaml new file mode 100644 index 000000000..dc066bbbf --- /dev/null +++ b/kubernetes/ansible/roles/logging/templates/oauth2-proxy.yaml @@ -0,0 +1,39 @@ +config: + clientID: "{{google_client_id}}" + clientSecret: "{{google_client_secret}}" + cookieSecret: "{{cookie_secret}}" + configFile: |- + redirect_url = "{{kibana_oauth_redirect_url}}" + upstreams = "http://{{ kibana_service }}" + email_domains = {{kibana_oauth_authenticated_email_domains_kube}} + authenticated_emails_file = "/etc/oauth2/auth-emails/authenticated_email.txt" + cookie_expire = "24h" + cookie_refresh = "4h" + cookie_secure = true + cookie_httponly = true + silence_ping_logging = true + skip_provider_button = true + +image: + repository: "sunbird/oauth2_proxy" + tag: "v1.0" + pullPolicy: "IfNotPresent" + +replicaCount: {{ oauth2_replicas | default(1) }} + +resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 108Mi + +authenticatedEmailsFile: + enabled: true + restricted_access: |- + {% for key in kibana_oauth_authenticated_email_ids %} + {{key}} + {% endfor %} + +fullnameOverride: "oauth2-proxy" diff --git a/kubernetes/ansible/roles/mount-keys/defaults/main.yml b/kubernetes/ansible/roles/mount-keys/defaults/main.yml new file mode 100644 index 000000000..9018a0d2d --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/defaults/main.yml @@ -0,0 +1,5 @@ +private_key_prefix: "*" +private_key_path: "keys" +private_key_sign_start: 0 +private_key_sign_end: 1 +private_key_sign_incr: 1 diff --git a/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm-public.yml b/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm-public.yml new file mode 100644 index 000000000..4320bccfd --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm-public.yml @@ -0,0 +1,14 @@ +--- +- name: Create the keys directory in case user is overriding the private_key_path + file: + path: "{{chart_path}}/keys" + state: directory + mode: 0755 + +- name: Copy keys directory to chart path + vars: + private_key_path: "{{private_key_path | regex_replace('^\\/|\\/$', '')}}" + copy: "src={{inventory_dir}}{{private_key_path}}{{private_key_prefix}}{{item}} dest={{chart_path}}/keys/" + with_sequence: start={{private_key_sign_start}} end={{private_key_sign_start|int + private_key_sign_end|int - 1}} stride={{private_key_sign_incr}} + when: (private_key_sign_start|int + private_key_sign_end|int - 1) > 0 + diff --git a/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm.yml b/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm.yml new file mode 100644 index 000000000..4320bccfd --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/copy-to-helm.yml @@ -0,0 +1,14 @@ +--- +- name: Create the keys directory in case user is overriding the private_key_path + file: + path: "{{chart_path}}/keys" + state: directory + mode: 0755 + +- name: Copy keys directory to chart path + vars: + private_key_path: "{{private_key_path | regex_replace('^\\/|\\/$', '')}}" + copy: "src={{inventory_dir}}{{private_key_path}}{{private_key_prefix}}{{item}} dest={{chart_path}}/keys/" + with_sequence: start={{private_key_sign_start}} end={{private_key_sign_start|int + private_key_sign_end|int - 1}} stride={{private_key_sign_incr}} + when: (private_key_sign_start|int + private_key_sign_end|int - 1) > 0 + diff --git a/kubernetes/ansible/roles/mount-keys/tasks/decrypt.yml b/kubernetes/ansible/roles/mount-keys/tasks/decrypt.yml new file mode 100644 index 000000000..db770f222 --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/decrypt.yml @@ -0,0 +1,7 @@ +--- +- name: Decrypt the data first + vars: + private_key_path: "{{private_key_path | regex_replace('^\\/|\\/$', '')}}" + shell: "ansible-vault decrypt {{ lookup('fileglob', '{{inventory_dir}}/{{private_key_path}}/{{private_key_prefix}}{{item}}') }} --vault-password-file /var/lib/jenkins/secrets/vault-pass" + with_sequence: start={{private_key_sign_start}} end={{private_key_sign_start|int + private_key_sign_end|int - 1}} stride={{private_key_sign_incr}} + when: (private_key_sign_start|int + private_key_sign_end|int - 1) > 0 diff --git a/kubernetes/ansible/roles/mount-keys/tasks/generate-keys.yml b/kubernetes/ansible/roles/mount-keys/tasks/generate-keys.yml new file mode 100644 index 000000000..fb7056609 --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/generate-keys.yml @@ -0,0 +1,7 @@ +--- +- name: Generate Public Keys for private keys + vars: + private_key_path: "{{private_key_path | regex_replace('^\\/|\\/$', '')}}" + shell: "openssl rsa -in {{ lookup('fileglob', '{{inventory_dir}}/{{private_key_path}}/{{private_key_prefix}}{{item}}') }} -outform PEM -pubout -out {{inventory_dir}}/{{private_key_path | regex_replace('^\\/|\\/$', '')}}/{{private_key_prefix}}{{item}}" + with_sequence: start={{private_key_sign_start}} end={{private_key_sign_start|int + private_key_sign_end|int - 1}} stride={{private_key_sign_incr}} + when: (private_key_sign_start|int + private_key_sign_end|int - 1) > 0 diff --git a/kubernetes/ansible/roles/mount-keys/tasks/onboard-keys.yml b/kubernetes/ansible/roles/mount-keys/tasks/onboard-keys.yml new file mode 100644 index 000000000..c9a33d3cf --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/onboard-keys.yml @@ -0,0 +1,15 @@ +--- +- name: Onboard the credentails using the generated public key + shell: 'curl -XPOST http://{{ private_ingressgateway_ip }}/admin-api/consumers/{{kong_consumer}}/jwt -F "key={{private_key_prefix}}{{item}}" -F "algorithm=RS256" -F "rsa_public_key=@{{inventory_dir}}/{{private_key_path}}/{{private_key_prefix}}{{item}}"' + args: + warn: false + with_sequence: start={{private_key_sign_start}} end={{private_key_sign_start|int + private_key_sign_end|int - 1}} stride={{private_key_sign_incr}} + when: private_key_sign_start is not none and private_key_sign_end is not none and private_key_user_input is not defined and (private_key_sign_start|int + private_key_sign_end|int - 1) > 0 + +- name: Onboard the credentails using the key number provided from Jenkins + shell: 'curl -XPOST http://{{ private_ingressgateway_ip }}/admin-api/consumers/{{kong_consumer}}/jwt -F "key={{private_key_prefix}}{{item}}" -F "algorithm=RS256" -F "rsa_public_key=@{{inventory_dir}}/{{private_key_path}}/{{private_key_prefix}}{{item}}"' + args: + warn: false + when: private_key_user_input is defined + with_items: + - "{{ private_key_user_input.split(',') }}" diff --git a/kubernetes/ansible/roles/mount-keys/tasks/remove-from-helm.yml b/kubernetes/ansible/roles/mount-keys/tasks/remove-from-helm.yml new file mode 100644 index 000000000..19fc5e545 --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/remove-from-helm.yml @@ -0,0 +1,5 @@ +--- +- name: Remove the keys directory post helm install + file: + path: "{{chart_path}}/keys" + state: absent diff --git a/kubernetes/ansible/roles/mount-keys/tasks/remove-keys-from-inventory.yml b/kubernetes/ansible/roles/mount-keys/tasks/remove-keys-from-inventory.yml new file mode 100644 index 000000000..1beade3f2 --- /dev/null +++ b/kubernetes/ansible/roles/mount-keys/tasks/remove-keys-from-inventory.yml @@ -0,0 +1,5 @@ +--- +- name: Remove the keys directory from inventory path as they are decrypted + file: + path: "{{inventory_dir}}/keys" + state: absent diff --git a/kubernetes/ansible/roles/proxy/README.md b/kubernetes/ansible/roles/proxy/README.md new file mode 100644 index 000000000..225dd44b9 --- /dev/null +++ b/kubernetes/ansible/roles/proxy/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/kubernetes/ansible/roles/proxy/defaults/main.yml b/kubernetes/ansible/roles/proxy/defaults/main.yml new file mode 100644 index 000000000..a857b4a8a --- /dev/null +++ b/kubernetes/ansible/roles/proxy/defaults/main.yml @@ -0,0 +1,4 @@ +--- +namespace: dev +merge_domain_status: false +merge_proxy_server_name: diff --git a/kubernetes/ansible/roles/proxy/tasks/main.yml b/kubernetes/ansible/roles/proxy/tasks/main.yml new file mode 100644 index 000000000..224f29a78 --- /dev/null +++ b/kubernetes/ansible/roles/proxy/tasks/main.yml @@ -0,0 +1,49 @@ +--- +# Proxy config +- name: Deleting domain ssl if present + shell: "kubectl get secrets ingress-cert -n istio-system && kubectl delete secrets ingress-cert -n istio-system" + ignore_errors: true + +- name: Creating domain sssl + no_log: true + shell: + cmd: | + cat < '0' else '1' }}" + when: release_name == "learner" or release_name == "lms" or release_name == "groups" + with_subelements: + - "{{adminutil_access_values}}" + - role_to_run + loop_control: + loop_var: outer_item + +- name: Create the token pubic key file + copy: + dest: "{{ chart_path }}/keys/{{ adminutil_refresh_token_public_key_kid }}" + content: "{{ core_vault_sunbird_sso_publickey }}" + when: release_name == "learner" or release_name == "lms" or release_name == "groups" + +- name: helm upgrade + shell: helm upgrade --install --atomic --timeout {{helm_install_timeout | d('10m')}} {{ release_name }} {{ chart_path }} -n {{namespace}} + +- name: Clean up keys + include_role: + name: mount-keys + tasks_from: "{{item}}" + when: release_name == "learner" or release_name == "lms" or release_name == "groups" + with_items: + - remove-from-helm.yml + - remove-keys-from-inventory.yml + +- name: Get the deployment rollout status + shell: "kubectl get deployments -A | grep -i {{ release_name }} | awk -F' ' '{print $3}' | awk -F/ '{if ($1 ~ $2){exit 0} else {exit 1}}'" + register: task_result + until: task_result.rc == 0 + retries: 6 + delay: 30 + +- name: Get deployed image name + shell: kubectl get deployments.apps {{ release_name }} -o json -n {{ namespace }} | jq -r '.spec.template.spec.containers[0].image | split("/")[1]' + register: deployed_image + +- set_fact: + metadata_image: "{{ image_name }}:{{ image_tag }}" + +- name: Compare deployed image and metadata_image + fail: + when: deployed_image.stdout != metadata_image diff --git a/kubernetes/ansible/roles/sunbird-monitoring/README.md b/kubernetes/ansible/roles/sunbird-monitoring/README.md new file mode 100644 index 000000000..9e3454265 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/README.md @@ -0,0 +1,49 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. +* monitor_alerts_mail_from_email: # Email from address +* monitor_alerts_mail_server_host: # Email host address +* monitor_alerts_mail_server_port: 587 # Email server port +* monitor_alerts_mail_server_username: # Email user +* monitor_alerts_mail_server_password: # Email password +* monitor_alerts_slack_url: # Slack URL +* monitor_alerts_slack_channel: # Slack channel +* env: # Environment name +* cluster_name: KUBERNETES # Cluster name +* default_mailing_list: # Default email list for alerts + + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml new file mode 100644 index 000000000..942d95e64 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml @@ -0,0 +1,212 @@ +--- +# defaults file for sunbird-monitoring +fullnameOverride: sunbird-monitoring +monitoring_stack: + - prometheus-operator + - cassandra-jmx-exporter + - elasticsearch-exporter + - logger + - blackbox-exporter + - additional-scrape-configs + - alertrules + - kafka-topic-exporter + - prometheus-redis-exporter + - processing-kafka-exporter + +namespace: monitoring + +monitor_alerts_mail_from_email: +monitor_alerts_mail_server_host: +monitor_alerts_mail_server_port: 587 +monitor_alerts_mail_server_username: +monitor_alerts_mail_server_password: +monitor_alerts_slack_url: +monitor_alerts_slack_channel: +env: dev +kubernetes_cluster_name: kubernetes-1 +#dp_monitor_alerts_slack_channel: "{{ monitor_alerts_slack_channel }}" +#dp_monitor_alerts_slack_url: "{{ core_vault_monitor_alerts_slack_url }}" +dp_monitor_alerts_warning_slack_url: "{{ core_vault_monitor_alerts_slack_url }}" +dp_monitor_alerts_warning_slack_channel: "{{ monitor_alerts_slack_channel }}" +dp_monitor_alerts_critical_slack_url: "{{ core_vault_monitor_alerts_slack_url }}" +dp_monitor_alerts_critical_slack_channel: "{{ monitor_alerts_slack_channel }}" + +default_critical_mailing_list: "{{ default_mailing_list }}" +default_fatal_mailing_list: "{{ default_mailing_list }}" +lp_process_exporter_team_mailing_list: "{{ default_mailing_list }}" +lp_process_exporter_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +lp_process_exporter_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" +druid_team_mailing_list: "{{ default_mailing_list }}" +druid_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +druid_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" +default_mailing_list: smy.altamash@gmail.com +dp_process_exporter_team_mailing_list: "{{ default_mailing_list }}" +dp_process_exporter_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +dp_process_exporter_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" +node_exporter_team_mailing_list: "{{ default_mailing_list }}" +node_exporter_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +node_exporter_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" +elasticsearch_exporter_team_mailing_list: "{{ default_mailing_list }}" +elasticsearch_exporter_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +elasticsearch_exporter_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" +blackbox_exporter_team_mailing_list: "{{ default_mailing_list }}" +blackbox_exporter_team_mailing_list_critical: "{{ default_critical_mailing_list }}" +blackbox_exporter_team_mailing_list_fatal: "{{ default_fatal_mailing_list }}" + +alert_teams: + - team: lp_process_exporter_team + group: groupname + alerts_mailing_list: "{{ lp_process_exporter_team_mailing_list }}" + services: + - tomcat_.* + - search_.* + - neo4j_.* + - kafka_.* + - zookeeper_.* + - yarn_.* + - redis-server.* + - cassandra_.* + - elasticsearch_.* + - logstash_.* + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ lp_process_exporter_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ lp_process_exporter_team_mailing_list_fatal }}" + - team: druid_process_exporter_team + group: groupname + alerts_mailing_list: "{{ druid_team_mailing_list }}" + services: + - druidzookeeper.* + - druidpostgres.* + - overlord.* + - coordinator.* + - historical.* + - broker.* + - middle-Manager.* + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ druid_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ druid_team_mailing_list_fatal }}" + - team: dp_process_exporter_team + group: groupname + alerts_mailing_list: "{{ dp_process_exporter_team_mailing_list }}" + services: + - analytics.* + - influx.* + - secor.* + - kafka_.* + - zookeeper_.* + - yarn_.* + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ dp_process_exporter_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ dp_process_exporter_team_mailing_list_fatal }}" + - team: node_exporter_team + group: job + alerts_mailing_list: "{{ node_exporter_team_mailing_list }}" + services: + - .*node-exporter + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ node_exporter_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ node_exporter_team_mailing_list_fatal }}" + - team: elasticsearch_exporter_team + group: service + alerts_mailing_list: "{{ elasticsearch_exporter_team_mailing_list }}" + services: + - .*elasticsearch-exporter + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ elasticsearch_exporter_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ elasticsearch_exporter_team_mailing_list_fatal }}" + - team: blackbox_exporter_team + group: service_name + alerts_mailing_list: "{{ blackbox_exporter_team_mailing_list }}" + services: + - .*-service + severity_mailing_filter: + - severity: critical + alerts_mailing_list: "{{ blackbox_exporter_team_mailing_list_critical }}" + - severity: fatal + alerts_mailing_list: "{{ blackbox_exporter_team_mailing_list_fatal }}" + +service_blackbox_checks: + - service_name: 'analytics-service' + probe_module: http_2xx + targets: + - "{{sunbird_analytics_api_base_url}}/health" + - service_name: 'learning-service' + probe_module: http_2xx + targets: + - "{{sunbird_content_repo_api_base_url}}/health" + - service_name: 'search-service' + probe_module: http_2xx + targets: + - "{{sunbird_search_service_api_base_url}}/health" + - service_name: 'keycloak-service' + probe_module: http_2xx + targets: + - "{{proto}}://{{proxy_server_name}}/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=foo&redirect_uri=https%3A%2F%2F{{proxy_server_name}}%2Fprivate%2Findex%3Fauth_callback%3D1&scope=openid&response_type=code" + +kafka_topic_prefix: sunbird +kafka_telemetry_assess_backup_warning: 50000 +kafka_telemetry_assess_raw_backup_threshold_warning: 50000 +content_consumption_events_backup_threshold_warning: 50000 +kafka_telemetry_denorm_backup_threshold_warning: 50000 +kafka_telemetry_derived_backup_threshold_warning: 50000 +kafka_events_deviceprofile_backup_threshold_warning: 50000 +kafka_telemetry_extractor_duplicate_backup_threshold_warning: 50000 +kafka_telemetry_extractor_failed_backup_threshold_warning: 50000 +kafka_telemetry_failed_backup_threshold_warning: 50000 +kafka_telemetry_ingestion_backup_threshold_warning: 50000 +kafka_graph_events_backup_threshold_warning: 50000 +kafka_learning_failed_events_backup_warning: 50000 +kafka_telemetry_raw_backup_threshold_warning: 50000 +kafka_telemetry_duplicate_backup_threshold_warning: 50000 +kafka_telemetry_ingestion_backup_threshold_warning: 50000 +kafka_telemetry_unique_backup_threshold_warning: 50000 +summary_backup_threshold_warning: 50000 +summary_channel_backup_threshold_warning: 50000 + + +kafka_telemetry_assess_backup_critical: 100000 +kafka_telemetry_assess_raw_backup_threshold_critical: 100000 +content_consumption_events_backup_threshold_critical: 100000 +kafka_telemetry_denorm_backup_threshold_critical: 100000 +kafka_telemetry_derived_backup_threshold_critical: 100000 +kafka_events_deviceprofile_backup_threshold_critical: 100000 +kafka_telemetry_extractor_duplicate_backup_threshold_critical: 100000 +kafka_telemetry_extractor_failed_backup_threshold_critical: 100000 +kafka_telemetry_failed_backup_threshold_critical: 100000 +kafka_telemetry_ingestion_backup_threshold_critical: 100000 +kafka_graph_events_backup_threshold_critical: 100000 +kafka_learning_failed_events_backup_critical: 100000 +kafka_telemetry_raw_backup_threshold_critical: 100000 +kafka_telemetry_duplicate_backup_threshold_critical: 100000 +kafka_telemetry_ingestion_backup_threshold_critical: 100000 +kafka_telemetry_unique_backup_threshold_critical: 100000 +summary_backup_threshold_critical: 100000 +summary_channel_backup_threshold_critical: 100000 + +### kafka exporters related vars +processing_cluster_zookeeper: "{{ groups['processing-cluster-zookeepers'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}" +processing_cluster_kafka: "{{ groups['processing-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}" + +ingestion_cluster_zookeeper: "{{ groups['ingestion-cluster-zookeeper'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}" +ingestion_cluster_kafka: "{{ groups['ingestion-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}" + +# These varaibles can be overridden in common.yaml +prometheus_spec_overrides: + sb_test_var: dummy_var_not_to_throw_error +alertmanager_spec_overrides: + sb_test_var: dummy_var_not_to_throw_error + +druid_raw_ingestion_lag_threshold: 200000 +druid_rollup_ingestion_lag_threshold: 200000 +rollup_druid_datasource: "" +redis_process_names: "" diff --git a/kubernetes/ansible/roles/sunbird-monitoring/handlers/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/handlers/main.yml new file mode 100644 index 000000000..b34ace797 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for sunbird-monitoring \ No newline at end of file diff --git a/kubernetes/ansible/roles/sunbird-monitoring/meta/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/meta/main.yml new file mode 100644 index 000000000..722379974 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/meta/main.yml @@ -0,0 +1,57 @@ +galaxy_info: + author: your name + description: your description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Some suggested licenses: + # - BSD (default) + # - MIT + # - GPLv2 + # - GPLv3 + # - Apache + # - CC-BY + license: license (GPLv2, CC-BY, etc) + + min_ansible_version: 1.2 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # Optionally specify the branch Galaxy will use when accessing the GitHub + # repo for this role. During role install, if no tags are available, + # Galaxy will use this branch. During import Galaxy will access files on + # this branch. If Travis integration is configured, only notifications for this + # branch will be accepted. Otherwise, in all cases, the repo's default branch + # (usually master) will be used. + #github_branch: + + # + # platforms is a list of platforms, and each platform has a name and a list of versions. + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. \ No newline at end of file diff --git a/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml new file mode 100644 index 000000000..8677a7f07 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml @@ -0,0 +1,21 @@ +--- +# tasks file for sunbird-monitoring +- name: tempating variables + template: + src: "{{ item }}.yaml" + dest: "/tmp/{{item}}.yaml" + with_items: "{{ monitoring_stack }}" + +- name: Creating sunbird monitoring stack + shell: "helm upgrade --install --atomic --timeout 10m {{ item }} {{chart_path}}/{{ item }} --namespace monitoring -f /tmp/{{ item }}.yaml" + with_items: "{{ monitoring_stack }}" + +- name: Creating sunbird monitoring grafana dashboards + shell: "helm upgrade --install grafana-dashboards {{chart_path}}/dashboards --namespace monitoring" + tags: + - dashboards + +- name: Install statsd-exporter + shell: "helm upgrade --install statsd-exporter {{chart_path}}/statsd-exporter --namespace {{ namespace }}" + tags: + - statsd-exporter diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/additional-scrape-configs.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/additional-scrape-configs.yaml new file mode 100644 index 000000000..e5eb3a5c7 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/additional-scrape-configs.yaml @@ -0,0 +1,100 @@ +# This name is used to define the +# additionalScrapeConfigs name +# {{ fullnameOverride }}-prometheus-scrape-confg +# If you change this, make sure to update the value in +# additionalScrapeConfigs/defautls/main.yaml +#} + +fullnameOverride: "sunbird-monitoring" + +scrapeconfig: + # This scrape config will enable us to annotate any pod and that pod will get automonitored + # + # annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "2020" + # prometheus.io/path: /api/v1/metrics/prometheus + # + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + + - job_name: 'Druid' + metrics_path: /metrics/druid + static_configs: + - targets: ["{{analyticsapi_ip}}:9000"] + labels: + "service_name": "druid" + + - job_name: 'vm-node-exporter' + static_configs: + - targets: ["{{ groups['node-exporter'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:9100') | list | join("\", \"") }}"] + + - job_name: 'process-exporter' + metrics_path: /metrics + static_configs: + - targets: ["{{ groups['process-exporter'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:1111' ) | list | join("\", \"") }}"] + + - job_name: 'cassandra-exporter' + metrics_path: /metrics + static_configs: + - targets: ["{{ groups['cassandra'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:8080' ) | list | join("\", \"") }}"] + +{% if 'druid' in groups and groups['druid'] %} + - job_name: 'druid-exporter' + metrics_path: /metrics + static_configs: + - targets: ["{{ groups['druid'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:9108' ) | list | join("\", \"") }}"] +{% endif %} + + - job_name: 'redis_exporter_targets' + static_configs: + - targets: ["{{ groups['redis-exporter-targets'] | difference(["localhost"]) | map('regex_replace', '^(.*)$', '\\1:6379') | list | join("\", \"") }}"] + metrics_path: /scrape + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: redis-exporter:9121 + + {% for item in service_blackbox_checks %} + # This empty line ensures indentation is correct after ansible jinja2 template is materialized + - job_name: 'availability_{{ item.service_name }}' + metrics_path: /metrics + params: + module: [{{ item.probe_module }}] + static_configs: + - targets: {{ item.targets | to_json }} + labels: + "service_name": "{{ item.service_name }}" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter-prometheus-blackbox-exporter:9115 + {% endfor %} diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml new file mode 100644 index 000000000..ed09e111c --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml @@ -0,0 +1,86 @@ +kafka_topic_prefix: "{{ env_name }}" +app_es_server_count: "{{ groups['es'] | length }}" +log_es_server_count: "{{ groups['log-es-2'] | length }}" +cassandra_server_count: "{{ groups['cassandra'] | length }}" +log_es_threshold: 30 +secor_job_count: "{{ secor_job_count | d(18) }}" +processingsecor_job_count: "{{ processingsecor_job_count | d(18) }}" + +kafka_telemetry_assess_backup_warning: "{{ kafka_telemetry_assess_backup_warning }}" +kafka_telemetry_assess_raw_backup_threshold_warning: "{{ kafka_telemetry_assess_raw_backup_threshold_warning }}" +content_consumption_events_backup_threshold_warning: "{{ content_consumption_events_backup_threshold_warning }}" +kafka_telemetry_denorm_backup_threshold_warning: "{{ kafka_telemetry_denorm_backup_threshold_warning }}" +kafka_telemetry_derived_backup_threshold_warning: "{{ kafka_telemetry_derived_backup_threshold_warning }}" +kafka_events_deviceprofile_backup_threshold_warning: "{{ kafka_events_deviceprofile_backup_threshold_warning }}" +kafka_telemetry_extractor_duplicate_backup_threshold_warning: "{{ kafka_telemetry_extractor_duplicate_backup_threshold_warning }}" +kafka_telemetry_extractor_failed_backup_threshold_warning: "{{ kafka_telemetry_extractor_failed_backup_threshold_warning }}" +kafka_telemetry_failed_backup_threshold_warning: "{{ kafka_telemetry_failed_backup_threshold_warning }}" +kafka_telemetry_ingestion_backup_threshold_warning: "{{ kafka_telemetry_ingestion_backup_threshold_warning }}" +kafka_graph_events_backup_threshold_warning: "{{ kafka_graph_events_backup_threshold_warning }}" +kafka_learning_failed_events_backup_warning: "{{ kafka_learning_failed_events_backup_warning }}" +kafka_telemetry_raw_backup_threshold_warning: "{{ kafka_telemetry_raw_backup_threshold_warning }}" +kafka_telemetry_duplicate_backup_threshold_warning: "{{ kafka_telemetry_duplicate_backup_threshold_warning }}" +kafka_telemetry_ingestion_backup_threshold_warning: "{{ kafka_telemetry_ingestion_backup_threshold_warning }}" +kafka_telemetry_unique_backup_threshold_warning: "{{ kafka_telemetry_unique_backup_threshold_warning }}" +summary_backup_threshold_warning: "{{ summary_backup_threshold_warning }}" +summary_channel_backup_threshold_warning: "{{ summary_channel_backup_threshold_warning }}" + +kafka_telemetry_assess_backup_critical: "{{ kafka_telemetry_assess_backup_critical }}" +kafka_telemetry_assess_raw_backup_threshold_critical: "{{ kafka_telemetry_assess_raw_backup_threshold_critical }}" +content_consumption_events_backup_threshold_critical: "{{ content_consumption_events_backup_threshold_critical }}" +kafka_telemetry_denorm_backup_threshold_critical: "{{ kafka_telemetry_denorm_backup_threshold_critical }}" +kafka_telemetry_derived_backup_threshold_critical: "{{ kafka_telemetry_derived_backup_threshold_critical }}" +kafka_events_deviceprofile_backup_threshold_critical: "{{ kafka_events_deviceprofile_backup_threshold_critical }}" +kafka_telemetry_extractor_duplicate_backup_threshold_critical: "{{ kafka_telemetry_extractor_duplicate_backup_threshold_critical }}" +kafka_telemetry_extractor_failed_backup_threshold_critical: "{{ kafka_telemetry_extractor_failed_backup_threshold_critical }}" +kafka_telemetry_failed_backup_threshold_critical: "{{ kafka_telemetry_failed_backup_threshold_critical }}" +kafka_telemetry_ingestion_backup_threshold_critical: "{{ kafka_telemetry_ingestion_backup_threshold_critical }}" +kafka_graph_events_backup_threshold_critical: "{{ kafka_graph_events_backup_threshold_critical }}" +kafka_learning_failed_events_backup_critical: "{{ kafka_learning_failed_events_backup_critical }}" +kafka_telemetry_raw_backup_threshold_critical: "{{ kafka_telemetry_raw_backup_threshold_critical }}" +kafka_telemetry_duplicate_backup_threshold_critical: "{{ kafka_telemetry_duplicate_backup_threshold_critical }}" +kafka_telemetry_ingestion_backup_threshold_critical: "{{ kafka_telemetry_ingestion_backup_threshold_critical }}" +kafka_telemetry_unique_backup_threshold_critical: "{{ kafka_telemetry_unique_backup_threshold_critical }}" +summary_backup_threshold_critical: "{{ summary_backup_threshold_critical }}" +summary_channel_backup_threshold_critical: "{{ summary_channel_backup_threshold_critical }}" + + +raw_druid_datasource: + - content-model-snapshot + - offline-desktop-events + - summary-events + - telemetry-events-syncts + - telemetry-feedback-events + - audit-rollup-syncts + - sl_observation_status + - sl_observations_production + - telemetry-feedback-events + +druid_raw_ingestion_threshold: "{{ druid_raw_ingestion_lag_threshold | default(200000) }}" +druid_rollup_ingestion_threshold: "{{ druid_rollup_ingestion_lag_threshold | default(200000)}}" +rollup_druid_datasource: + {{ rollup_druid_datasource | to_yaml | indent( width=2) }} + +secor_process_names: + - channel-telemetry-backup + - derived-telemetry-backup + - derived-denorm-events-backup + - channel-summary-backup + - assess-events-backup + - assess-raw-events-backup + - device-profile-backup + - learning-events-backup + - learning-failed-backup + - content-consumption-events-backup + - failed-telemetry-backup + - extractor-duplicate-backup + - extractor-failed-backup + - unique-telemetry-backup + - denorm-events-backup + - raw-telemetry-backup + - ingestion-telemetry-backup + - ingestion-cluster-telemetry-backup + - telemetry-ingest-backup + +redis_process_names: + {{ redis_process_names | to_yaml | indent( width=2) }} diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/blackbox-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/blackbox-exporter.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/cassandra-jmx-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/cassandra-jmx-exporter.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/elasticsearch-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/elasticsearch-exporter.yaml new file mode 100644 index 000000000..e3a60ff61 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/elasticsearch-exporter.yaml @@ -0,0 +1,11 @@ +es: + uri: "http://{{ groups['es-1'][0] }}:9200" +serviceMonitor: + enabled: true + # namespace: monitoring + labels: # labels with which the prometheus choose the serviceMonitor + app: monitoring + release: prometheus-operator + interval: 60s + scrapeTimeout: 30s + scheme: http diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/ingestion-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/ingestion-kafka-exporter.yaml new file mode 100644 index 000000000..7ddffd74e --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/ingestion-kafka-exporter.yaml @@ -0,0 +1,17 @@ +kafkaExporter: + zookeeper: + servers: ["{{ ingestion_cluster_zookeeper | join('","') }}"] + kafka: + servers: ["{{ ingestion_cluster_kafka | join('","') }}"] + additionalFlags: + - --use.consumelag.zookeeper + +prometheus: + serviceMonitor: + enabled: true + namespace: monitoring + interval: "120s" + scrapeTimeout: "90s" + additionalLabels: + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-lag-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-lag-exporter.yaml new file mode 100644 index 000000000..22cd19d2a --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-lag-exporter.yaml @@ -0,0 +1,8 @@ +kafka: "{{ groups['processing-cluster-kafka'] | join(':9092,')}}:9092" +zookeeper: "{{ groups['processing-cluster-kafka'] | join(':2181,')}}:2181" +# This section will create service monitor for prometheus operator +serviceMonitor: + enabled: true + labels: # labels with which the prometheus choose the serviceMonitor + app: monitoring + release: prometheus-operator diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-topic-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-topic-exporter.yaml new file mode 100644 index 000000000..2c3defc8b --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/kafka-topic-exporter.yaml @@ -0,0 +1,34 @@ +env: + kafka: + host: "{{ kafka_topic_exporter_hosts | d(groups['kafka']|join(':9092,')+':9092') }}" + topic: "{{ kafka_topic_exporter_topics | d(env_name + '.prom.monitoring.metrics') }}" + +#kafka-critical lag variables +assessment_aggregator_threshold_critical: "{{ assessment_aggregator_threshold_critical | d(100) }}" +deduplication_threshold_critical: "{{ deduplication_threshold_critical | d(100) }}" +denormalization_threshold_critical: "{{ denormalization_threshold_critical | d(100) }}" +derived_deduplication_threshold_critical: "{{ derived_deduplication_threshold_critical | d(100) }}" +device_profileupdater_threshold_critical: "{{ device_profileupdater_threshold_critical | d(100) }}" +druidevents_validator_threshold_critical: "{{ druidevents_validator_threshold_critical | d(100) }}" +events_router_threshold_critical: "{{ events_router_threshold_critical | d(100) }}" +telemetry_extractor_threshold_critical: "{{ telemetry_extractor_threshold_critical | d(100) }}" +telemetry_location_updater_threshold_critical: "{{ telemetry_location_updater_threshold_critical | d(100) }}" +telemetry_redacter_threshold_critical: "{{ telemetry_redacter_threshold_critical | d(100) }}" +telemetry_router_threshold_critical: "{{ telemetry_router_threshold_critical | d(100) }}" +telemetry_validator_threshold_critical: "{{ telemetry_validator_threshold_critical | d(100) }}" +user_cacheupdater_threshold_critical: "{{ user_cacheupdater_threshold_critical | d(100) }}" + +#kafka-fatal lag variables +assessment_aggregator_threshold_fatal: "{{ assessment_aggregator_threshold_fatal | d(500) }}" +deduplication_threshold_fatal: "{{ deduplication_threshold_fatal | d(500) }}" +denormalization_threshold_fatal: "{{ denormalization_threshold_fatal | d(500) }}" +derived_deduplication_threshold_fatal: "{{ derived_deduplication_threshold_fatal | d(500) }}" +device_profileupdater_threshold_fatal: "{{ device_profileupdater_threshold_fatal | d(500) }}" +druidevents_validator_threshold_fatal: "{{ druidevents_validator_threshold_fatal | d(500) }}" +events_router_threshold_fatal: "{{ events_router_threshold_fatal | d(500) }}" +telemetry_extractor_threshold_fatal: "{{ telemetry_extractor_threshold_fatal | d(500) }}" +telemetry_location_updater_threshold_fatal: "{{ telemetry_location_updater_threshold_fatal | d(500) }}" +telemetry_redacter_threshold_fatal: "{{ telemetry_redacter_threshold_fatal | d(500) }}" +telemetry_router_threshold_fatal: "{{ telemetry_router_threshold_fatal | d(500) }}" +telemetry_validator_threshold_fatal: "{{ telemetry_validator_threshold_fatal | d(500) }}" +user_cacheupdater_threshold_fatal: "{{ user_cacheupdater_threshold_fatal | d(500) }}" diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/logger.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/logger.yaml new file mode 100644 index 000000000..2b27868e1 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/logger.yaml @@ -0,0 +1,11 @@ +es: + uri: "http://{{ groups['log-es-2'][0] }}:9200" +serviceMonitor: + enabled: true + # namespace: monitoring + labels: # labels with which the prometheus choose the serviceMonitor + app: monitoring + release: prometheus-operator + interval: 60s + scrapeTimeout: 30s + scheme: http diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/processing-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/processing-kafka-exporter.yaml new file mode 100644 index 000000000..69d64410b --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/processing-kafka-exporter.yaml @@ -0,0 +1,17 @@ +kafkaExporter: + zookeeper: + servers: ["{{ processing_cluster_zookeeper | join('","') }}"] + kafka: + servers: ["{{ processing_cluster_kafka | join('","') }}"] + additionalFlags: + - --use.consumelag.zookeeper + +prometheus: + serviceMonitor: + enabled: true + namespace: monitoring + interval: "120s" + scrapeTimeout: "90s" + additionalLabels: + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml new file mode 100644 index 000000000..ce05bde3a --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml @@ -0,0 +1,272 @@ +#jinja2:lstrip_blocks: True +{# +# This name is used to define the +# additionalScrapeConfigs name +# {{ fullnameOverride }}-prometheus-scrape-confg +# If you change this, make sure to update the value in +# additionalScrapeConfigs/defautls/main.yaml +#} +fullnameOverride: sunbird-monitoring + +# This is to override/add configurations of prometheusSpec by user +# you can add this variable in common.yaml and that will get merged / preferred +# over the below variables. + +# for example: +# +# prometheus_spec_overrides: +# additionalAlertManagerConfigs: +# - static_configs: +# - targets: +# - sunbird-monitoring-alertmanager.monitoring.svc.cluster.local:9093 +# - 28.0.33.125:9093 +# additionalAlertRelabelConfigs: +# - source_labels: [sb_cluster] +# regex: (.+)\d+ +# target_label: sb_cluster + + +# yaml anchor for prometheusspec over ride +prometheus_spec_overrides: &prometheus_spec_overrides + {{ prometheus_spec_overrides | to_yaml | indent( width=2) }} + +# For Example: +# +# alertmanager_spec_overrides: +# service: +# annotations: +# service.beta.kubernetes.io/azure-load-balancer-internal: "true" +# type: LoadBalancer +# # make sure this ip is in valid ip range +# # For aws this won't work, as aws will only give domain_name +# loadBalancerIP: "10.0.0.10" + +# Yaml anchor for alertmanager +alertmanager_spec_overrides: &alertmanager_spec_overrides + {{ alertmanager_spec_overrides | to_yaml | indent( width=2) }} + + +# Enabling external prometheus scrape config +prometheus: + prometheusSpec: + thanos: + version: {{ thanos_sidecar_version | d('v0.11.0') }} + additionalScrapeConfigsExternal: true + retention: "{{ prometheus_retention_time | d('90d') }}" + externalLabels: + cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}" # this label will deprecate in release-3.4.0 + env: "{{ env }}" # this label will deprecate in release-3.4.0 + sb_cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}" + sb_env: "{{ env }}" +{% if prometheus_storage_spec is defined and prometheus_storage_spec %} + storageSpec: {{ prometheus_storage_spec|to_json }} +{% endif %} + # Adding prometeus custom spec overrides + # Refering promtheus sepc override yaml anchor + <<: *prometheus_spec_overrides +{% if prometheus_service is defined and prometheus_service %} + service: {{ (prometheus_service | to_json) }} +{% endif %} + +alertmanager: + config: + global: + smtp_from: "{{ monitor_alerts_mail_from_email }}" + smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}" + smtp_auth_username: "{{ monitor_alerts_mail_server_username }}" + smtp_auth_password: "{{ monitor_alerts_mail_server_password }}" + + route: + receiver: '{{env}}_devops_team' + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + routes: + - match: + module: dp_process + receiver: dp-process_slack + - match: + module: dp_lag + severity: warning + receiver: dp-lag_slack_warning + - match: + module: dp_lag + severity: critical + receiver: dp-lag_slack_critical + - receiver: slack + continue: true + {% for item in alert_teams %} + # Comment to ensure proper indentation while templating + - match_re: + {{ item.group }}: "{{ item.services | join('|') }}" + receiver: "{{ item.team }}" + # Comment to ensure proper indentation while templating + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + routes: + {% for filter in item.severity_mailing_filter %} + # Comment to ensure proper indentation while templating + - match: + severity: "{{ filter.severity }}" + receiver: "{{ item.team }}_{{ filter.severity }}" + {% endfor %} + {% endif %} + {% endfor %} + # Comment to ensure proper indentation while templating + - match: + owner: + receiver: {{env}}_devops_team + + receivers: + - name: 'slack' + slack_configs: + - send_resolved: true + api_url: "{{ monitor_alerts_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ monitor_alerts_slack_channel }}" + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.message {{ "}}" }} - `{{ "{{" }} .Labels.severity {{ "}}" }}` + *Description:* {{ "{{" }} .Annotations.message {{ "}}" }} + *Details:* + {{ "{{" }} range .Labels.SortedPairs {{ "}}" }} • *{{ "{{" }} .Name {{ "}}" }}:* `{{ "{{" }} .Value {{ "}}" }}` + {{ "{{" }} end {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + - name: 'dp-process_slack' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_critical_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_critical_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *ProcessName:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'dp-lag_slack_warning' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_warning_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_warning_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + - name: 'dp-lag_slack_critical' + slack_configs: + - send_resolved: true + api_url: "{{ dp_monitor_alerts_critical_slack_url }}" + username: 'Monitor - Alerter' + channel: "{{ dp_monitor_alerts_critical_slack_channel }}" + title_link: "" + title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' + text: |- + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }} + *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }} + *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }} + *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }} + *Details:* {{ "{{" }} .Annotations.message {{ "}}" }} + {{ "{{" }} end {{ "}}" }} + icon_emoji: ':dart:' + + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + {% for item in alert_teams %} + # Comment to ensure proper indentation while templating + - name: "{{ item.team }}" + email_configs: + - send_resolved: true + to: '{{ item.alerts_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% if item.severity_mailing_filter is defined and item.severity_mailing_filter|length %} + {% for filter in item.severity_mailing_filter %} + # Comment to ensure proper indentation while templating + - name: "{{ item.team }}_{{ filter.severity }}" + email_configs: + - send_resolved: true + to: '{{ filter.alerts_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + {% endfor %} + {% endif %} + {% endfor %} + # Comment to ensure proper indentation while templating + - name: {{env}}_devops_team + email_configs: + - send_resolved: true + to: '{{ default_mailing_list }}' + html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}' + headers: + subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}' + + # Adding alertmanager custom spec overrides + # Refrencing alert manager yaml anchor + <<: *alertmanager_spec_overrides + + +grafana: + image: + repository: grafana/grafana + tag: 7.0.6 + pullPolicy: IfNotPresent + env: + GF_SERVER_ROOT_URL: http://grafana.local.com/grafana + adminPassword: {{ (grafana_admin_password| default('prom-operator'))}} +{% if grafana_data_sources is defined and grafana_data_sources %} + additionalDataSources: {{ (grafana_data_sources) | to_json}} +{% endif %} +{% if grafana_persistence is defined and grafana_persistence %} + persistence: {{ grafana_persistence|to_json}} +{% endif %} +{% if grafana_service is defined and grafana_service %} + service: {{ grafana_service|to_json}} +{% endif %} + + +# Override anything from common.yaml +# This will override the entire dictionary which we define above +# Means if you define a key which define above, you'll have to redifine +# entire dictionary here +{% if prometheus_additional_configs is defined and prometheus_additional_configs %} +{{ ( prometheus_additional_configs | to_yaml ) }} +{% endif %} diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-redis-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-redis-exporter.yaml new file mode 100644 index 000000000..f900e9484 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-redis-exporter.yaml @@ -0,0 +1 @@ +fullnameOverride: redis-exporter diff --git a/kubernetes/ansible/roles/sunbird-monitoring/tests/inventory b/kubernetes/ansible/roles/sunbird-monitoring/tests/inventory new file mode 100644 index 000000000..878877b07 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/kubernetes/ansible/roles/sunbird-monitoring/tests/test.yml b/kubernetes/ansible/roles/sunbird-monitoring/tests/test.yml new file mode 100644 index 000000000..5f63280f1 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - sunbird-monitoring \ No newline at end of file diff --git a/kubernetes/ansible/roles/sunbird-monitoring/vars/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/vars/main.yml new file mode 100644 index 000000000..d22cd7503 --- /dev/null +++ b/kubernetes/ansible/roles/sunbird-monitoring/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for sunbird-monitoring \ No newline at end of file diff --git a/kubernetes/ansible/static-files/api-manager.env b/kubernetes/ansible/static-files/api-manager.env new file mode 100644 index 000000000..ac808b0a5 --- /dev/null +++ b/kubernetes/ansible/static-files/api-manager.env @@ -0,0 +1,7 @@ +KONG_LOG_LEVEL=info +KONG_DATABASE={{ kong_database|d('postgres') }} +KONG_PG_HOST={{ kong_postgres_host }} +KONG_PG_SSL={{ kong_ssl|d('true') }} +KONG_PG_USER={{ kong_postgres_user }} +KONG_PG_PASSWORD={{ kong_postgres_password }} +KONG_PG_DATABASE={{ kong_postgres_database }} diff --git a/kubernetes/ansible/static-files/api_count_query.sh b/kubernetes/ansible/static-files/api_count_query.sh new file mode 100755 index 000000000..ce22a2549 --- /dev/null +++ b/kubernetes/ansible/static-files/api_count_query.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +###-----------------------------------------------------### +# Author:: Kaliraja +# Description:: This script is to query the api calls data +# from log-es and send as a report to email and uploading +# the same to azure storage. +###-----------------------------------------------------### + + +#date variables +prev_day=`date "+%s" -d "yesterday 03:30:00"` +today=`date "+%s" -d "today 03:30:00"` +date=`date +"%m-%d-%Y"` + +#prev_day=`date "+%s" -d "yesterday -7 day 03:30:00"` +#today=`date "+%s" -d "yesterday -6 day 03:30:00"` +#date=`date +%m-%d-%y --date="yesterday -6 day" | sed 's/19/2019/'` + +#api variable +contentsearch="/api/composite/v1/search" +contentread="/api/content/v1/read/" +telemetry="/api/data/v1/telemetry" +registermobile="/api/api-manager/v1/consumer/mobile_device/credential/register" + +#filename variable +contentsearch_filename=contentsearch-$date.txt +contentread_filename=contentread-$date.txt +telemetry_filename=telemetry-$date.txt +mobiledevice_registerfilename=registermobile-$date.txt + +#sedngrid variable +sguser="$1" +sgpass="$2" +container_name="$3" +account_name="$4" +storage_key="$5" + + +query(){ + curl -H 'Content-Type:application/json' -s -XPOST 'localhost:9200/logstash-*/_search?pretty' -d '{"query":{"bool":{"must":{"query_string":{"analyze_wildcard":true,"query":"\"'$1'\""}},"filter":{"bool":{"must":[{"range":{"@timestamp":{"gte":"'"$prev_day"'","lte":"'"$today"'","format":"epoch_second"}}}],"must_not":[]}}}},"size":0,"aggs":{"2":{"date_histogram":{"field":"@timestamp","interval":"15m","time_zone":"Asia/Kolkata","min_doc_count":1,"extended_bounds":{"min": 0,"max": 500}}}}}' | jq -r '.aggregations."2".buckets[]|.key_as_string+" "+ (.doc_count|tostring)' | column -t > $2 +} + +#Executing content search query + +query $contentsearch $contentsearch_filename + +#Execurting the contentread query + +query $contentread $contentread_filename + +#Executing the telemetry query + +query $telemetry $telemetry_filename + +#Executing the registermobiledevice query + +query $registermobile $mobiledevice_registerfilename + +#sending an email with an attachment + +curl https://api.sendgrid.com/api/mail.send.json \ + {{ api_report_mailing_list }} -F subject="Data for Diksha api calls" \ + -F text="Data" --form-string html="Hi Team, PFA." \ + -F from=reports@diksha.in -F api_user="$sguser" -F api_key="$sgpass" \ + -F files\[contentsearch.txt\]=@contentsearch-$date.txt -F files\[contentread.txt\]=@contentread-$date.txt -F files\[telemetry.txt]=@telemetry-$date.txt -F files\[registermobile.txt]=@registermobile-$date.txt + + +# uploading the reports to storage + +az storage blob upload \ +--container-name $container_name \ +--file contentsearch-$date.txt \ +--name contentsearch-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + +az storage blob upload \ +--container-name $container_name \ +--file contentread-$date.txt \ +--name contentread-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + + +az storage blob upload \ +--container-name $container_name \ +--file telemetry-$date.txt \ +--name telemetry-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + +az storage blob upload \ +--container-name $container_name \ +--file registermobile-$date.txt \ +--name registermobile-$date.txt \ +--account-name $account_name \ +--account-key $storage_key + + +# deleting files + +rm *-$date.txt diff --git a/kubernetes/ansible/static-files/elasticsearch_mapping.sh b/kubernetes/ansible/static-files/elasticsearch_mapping.sh new file mode 100644 index 000000000..ded56118f --- /dev/null +++ b/kubernetes/ansible/static-files/elasticsearch_mapping.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +#### Cloning and Changing the directory: +rm -rf sunbird-utils +git clone https://github.com/project-sunbird/sunbird-utils.git -b $1 +rm sunbird-utils/sunbird-es-utils/src/main/resources/indices/cbatchstats.json +rm sunbird-utils/sunbird-es-utils/src/main/resources/mappings/cbatchstats-mapping.json +cd sunbird-utils/sunbird-es-utils/src/main/resources/indices + +#### Creating the new indices: + +indices_files=$(ls -l | awk 'NR>1{print $9}' | awk -F"." '{print $1}' | tr "\n" " ") +for file in ${indices_files[@]} +do + curl -X PUT http://localhost:9200/${file} -H 'Content-Type: application/json' -d @${file}.json +done + +#### Updating the mapping for newly created indices: + +echo "#################################################" + +cd ../mappings +#mapping_files=$(ls -l | awk 'NR>1{print $9}' | awk -F"-" '{print $1}' | tr "\n" " ") +mapping_files=$(ls -l | awk 'NR>1{print $9}' | awk -F"." '{print $1}' | tr "\n" " " | sed 's/-mapping//g') + +for file in ${mapping_files[@]} +do + curl -X PUT http://localhost:9200/${file}/_mapping/_doc -H 'Content-Type: application/json' -d @${file}-mapping.json +done diff --git a/kubernetes/ansible/static-files/health.sh b/kubernetes/ansible/static-files/health.sh new file mode 100755 index 000000000..575fd669e --- /dev/null +++ b/kubernetes/ansible/static-files/health.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +#apk update curl +#apk add curl +#apk update jq +#apk add jq +outpt1=$(curl -s content-service:5000/health | jq '.result.healthy') +outpt2=$(curl -s player_player:3000/health| jq '.result.healthy') +outpt3=$(curl -s learner-service:9000/health | jq '.result.response.checks[0].healthy') +outpt4=$(curl -s lms-service:9005/health | jq '.result.response.checks[0].healthy') +echo "" +echo "" +if [ "$outpt1" == "true" ];then + echo "content service is Healthy" +else + echo "content service is unhealthy" +fi + +echo "" +echo "" + +if [ "$outpt2" == "true" ];then + echo "Player Service is Healthy" +else + echo "Player Service is unhealthy" +fi + +echo "" +echo "" + +if [ "$outpt3" == "true" ];then + echo "Learner Service is Healthy" +else + echo "Learner Service is unhealthy" +fi + +echo "" +echo "" + +if [ "$outpt4" == "true" ];then + echo "Lms Service is Healthy" +else + echo "Lms Service is unhealthy" +fi diff --git a/kubernetes/ansible/static-files/healthDependenices.sh b/kubernetes/ansible/static-files/healthDependenices.sh new file mode 100755 index 000000000..db3e0e219 --- /dev/null +++ b/kubernetes/ansible/static-files/healthDependenices.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +apk update curl +apk add curl +apk update jq +apk add jq diff --git a/kubernetes/ansible/static-files/kong-api-scripts/__init__.py b/kubernetes/ansible/static-files/kong-api-scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kubernetes/ansible/static-files/kong-api-scripts/common.py b/kubernetes/ansible/static-files/kong-api-scripts/common.py new file mode 100644 index 000000000..c001bac1a --- /dev/null +++ b/kubernetes/ansible/static-files/kong-api-scripts/common.py @@ -0,0 +1,41 @@ +import urllib2, json, logging +from retry import retry + +logging.basicConfig() + +# Due to issue https://github.com/Mashape/kong/issues/1912 +# We can't loop through all apis page by page +# Hence this work around which fetches apis with page size limited to max_page_size +# max_page_size ensures we don't bring down DB by fetching lot of rows +# If we reach a state we have more apis than max_page_size, +# Increase value of max_page_size judiciously +def get_apis(kong_admin_api_url): + max_page_size = 2000 + apis_url_with_size_limit = "{}/apis?size={}".format(kong_admin_api_url, max_page_size) + apis_response = json.loads(retrying_urlopen(apis_url_with_size_limit).read()) + total_apis = apis_response["total"] + if(total_apis > max_page_size): + raise Exception("There are {} apis existing in system which is more than max_page_size={}. Please increase max_page_size in ansible/kong_apis.py if this is expected".format(total_apis, max_page_size)) + else: + return apis_response["data"] + +def get_api_plugins(kong_admin_api_url, api_name): + get_plugins_max_page_size = 2000 + api_pugins_url = "{}/apis/{}/plugins".format(kong_admin_api_url, api_name) + get_api_plugins_url = "{}?size={}".format(api_pugins_url, get_plugins_max_page_size) + saved_api_details = json.loads(retrying_urlopen(get_api_plugins_url).read()) + return saved_api_details["data"] + + +def json_request(method, url, data=None): + request_body = json.dumps(data) if data is not None else None + request = urllib2.Request(url, request_body) + if data: + request.add_header('Content-Type', 'application/json') + request.get_method = lambda: method + response = retrying_urlopen(request) + return response + +@retry(exceptions=urllib2.URLError, tries=5, delay=2, backoff=2) +def retrying_urlopen(*args, **kwargs): + return urllib2.urlopen(*args, **kwargs) \ No newline at end of file diff --git a/kubernetes/ansible/static-files/kong-api-scripts/kong_api_csv_to_yaml.py b/kubernetes/ansible/static-files/kong-api-scripts/kong_api_csv_to_yaml.py new file mode 100644 index 000000000..5b886ef1d --- /dev/null +++ b/kubernetes/ansible/static-files/kong-api-scripts/kong_api_csv_to_yaml.py @@ -0,0 +1,37 @@ +import argparse, sys +from collections import OrderedDict +import csv +import yaml + +def setup_yaml(): + """ https://stackoverflow.com/a/31609484/69362 """ + represent_dict_order = lambda self, data: self.represent_mapping('tag:yaml.org,2002:map', data.items()) + yaml.add_representer(OrderedDict, represent_dict_order) + +def convert_csv_to_yaml(apis_csv_file): + reader = csv.DictReader(apis_csv_file, delimiter=',') + apis = [] + for row in reader: + apis.append(OrderedDict([ + ('name', row['NAME']), + ('request_path', row['REQUEST PATH']), + ('upstream_url', row['UPSTREAM PATH']), + ('strip_request_path', True), + ('plugins', [ + OrderedDict([('name', 'jwt')]), + OrderedDict([('name', 'cors')]), + "{{ statsd_pulgin }}", + OrderedDict([('name', 'acl'), ('config.whitelist', row["WHITELIST GROUP"])]), + OrderedDict([('name', 'rate-limiting'), ('config.hour', row["RATE LIMIT"]), ('config.limit_by', row["LIMIT BY"])]), + OrderedDict([('name', 'request-size-limiting'), ('config.allowed_payload_size', row["REQUEST SIZE LIMIT"])]), + ]) + ])) + yaml.dump(apis, sys.stdout, default_flow_style=False) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Converts APIs CSV to yaml that can be used in ansible') + parser.add_argument('apis_csv_file_path', help='Path of the csv file containing apis data') + args = parser.parse_args() + setup_yaml() + with open(args.apis_csv_file_path) as apis_csv_file: + convert_csv_to_yaml(apis_csv_file) diff --git a/kubernetes/ansible/static-files/kong-api-scripts/kong_apis.py b/kubernetes/ansible/static-files/kong-api-scripts/kong_apis.py new file mode 100644 index 000000000..9936160f9 --- /dev/null +++ b/kubernetes/ansible/static-files/kong-api-scripts/kong_apis.py @@ -0,0 +1,86 @@ +import urllib2, argparse, json + +from common import get_apis, json_request, get_api_plugins + +def save_apis(kong_admin_api_url, input_apis): + apis_url = "{}/apis".format(kong_admin_api_url) + saved_apis = get_apis(kong_admin_api_url) + + print("Number of input APIs : {}".format(len(input_apis))) + print("Number of existing APIs : {}".format(len(saved_apis))) + + input_api_names = [api["name"] for api in input_apis] + saved_api_names = [api["name"] for api in saved_apis] + + print("Input APIs : {}".format(input_api_names)) + print("Existing APIs : {}".format(saved_api_names)) + + input_apis_to_be_created = [input_api for input_api in input_apis if input_api["name"] not in saved_api_names] + input_apis_to_be_updated = [input_api for input_api in input_apis if input_api["name"] in saved_api_names] + saved_api_to_be_deleted = [saved_api for saved_api in saved_apis if saved_api["name"] not in input_api_names] + + for input_api in input_apis_to_be_created: + print("Adding API {}".format(input_api["name"])) + json_request("POST", apis_url, _sanitized_api_data(input_api)) + + for input_api in input_apis_to_be_updated: + print("Updating API {}".format(input_api["name"])) + saved_api_id = [saved_api["id"] for saved_api in saved_apis if saved_api["name"] == input_api["name"]][0] + input_api["id"] = saved_api_id + json_request("PATCH", apis_url + "/" + saved_api_id, _sanitized_api_data(input_api)) + + for saved_api in saved_api_to_be_deleted: + print("Deleting API {}".format(saved_api["name"])); + json_request("DELETE", apis_url + "/" + saved_api["id"], "") + + for input_api in input_apis: + _save_plugins_for_api(kong_admin_api_url, input_api) + +def _save_plugins_for_api(kong_admin_api_url, input_api_details): + get_plugins_max_page_size = 2000 + api_name = input_api_details["name"] + input_plugins = input_api_details["plugins"] + api_pugins_url = "{}/apis/{}/plugins".format(kong_admin_api_url, api_name) + saved_plugins_including_consumer_overrides = get_api_plugins(kong_admin_api_url, api_name) + saved_plugins_without_consumer_overrides = [plugin for plugin in saved_plugins_including_consumer_overrides if not plugin.get('consumer_id')] + + saved_plugins = saved_plugins_without_consumer_overrides + input_plugin_names = [input_plugin["name"] for input_plugin in input_plugins] + saved_plugin_names = [saved_plugin["name"] for saved_plugin in saved_plugins] + + input_plugins_to_be_created = [input_plugin for input_plugin in input_plugins if input_plugin["name"] not in saved_plugin_names] + input_plugins_to_be_updated = [input_plugin for input_plugin in input_plugins if input_plugin["name"] in saved_plugin_names] + saved_plugins_to_be_deleted = [saved_plugin for saved_plugin in saved_plugins if saved_plugin["name"] not in input_plugin_names] + + for input_plugin in input_plugins_to_be_created: + print("Adding plugin {} for API {}".format(input_plugin["name"], api_name)); + json_request("POST", api_pugins_url, input_plugin) + + for input_plugin in input_plugins_to_be_updated: + print("Updating plugin {} for API {}".format(input_plugin["name"], api_name)); + saved_plugin_id = [saved_plugin["id"] for saved_plugin in saved_plugins if saved_plugin["name"] == input_plugin["name"]][0] + input_plugin["id"] = saved_plugin_id + json_request("PATCH", api_pugins_url + "/" + saved_plugin["id"], input_plugin) + + for saved_plugin in saved_plugins_to_be_deleted: + print("Deleting plugin {} for API {}".format(saved_plugin["name"], api_name)); + json_request("DELETE", api_pugins_url + "/" + saved_plugin["id"], "") + +def _sanitized_api_data(input_api): + keys_to_ignore = ['plugins'] + sanitized_api_data = dict((key, input_api[key]) for key in input_api if key not in keys_to_ignore) + return sanitized_api_data + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Configure kong apis') + parser.add_argument('apis_file_path', help='Path of the json file containing apis data') + parser.add_argument('--kong-admin-api-url', help='Admin url for kong', default='http://localhost:8001') + args = parser.parse_args() + with open(args.apis_file_path) as apis_file: + input_apis = json.load(apis_file) + try: + save_apis(args.kong_admin_api_url, input_apis) + except urllib2.HTTPError as e: + error_message = e.read() + print(error_message) + raise \ No newline at end of file diff --git a/kubernetes/ansible/static-files/kong-api-scripts/kong_apis_report.py b/kubernetes/ansible/static-files/kong-api-scripts/kong_apis_report.py new file mode 100644 index 000000000..6306ae920 --- /dev/null +++ b/kubernetes/ansible/static-files/kong-api-scripts/kong_apis_report.py @@ -0,0 +1,20 @@ +import urllib2, argparse, json, csv + +from common import get_apis, json_request + +def create_api_report_csv(kong_admin_api_url, report_file_path): + saved_apis = get_apis(kong_admin_api_url) + with open(report_file_path, 'w') as csvfile: + fieldnames = ['Name', 'Path'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + writer.writeheader() + for api in saved_apis: + writer.writerow({'Name': api['name'], 'Path': api['request_path']}) + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate report for APIs on-boarded') + parser.add_argument('report_file_path', help='Report file path') + parser.add_argument('--kong-admin-api-url', help='Admin url for kong', default='http://localhost:8001') + args = parser.parse_args() + + create_api_report_csv(args.kong_admin_api_url, args.report_file_path) diff --git a/kubernetes/ansible/static-files/kong-api-scripts/kong_consumers.py b/kubernetes/ansible/static-files/kong-api-scripts/kong_consumers.py new file mode 100644 index 000000000..3b396ce07 --- /dev/null +++ b/kubernetes/ansible/static-files/kong-api-scripts/kong_consumers.py @@ -0,0 +1,163 @@ +import urllib2, argparse, json +import jwt + +from common import json_request, get_api_plugins, retrying_urlopen + +def _consumer_exists(kong_admin_api_url, username): + consumers_url = "{}/consumers".format(kong_admin_api_url) + try: + retrying_urlopen(consumers_url + "/" + username) + return True + except urllib2.HTTPError as e: + if(e.code == 404): + return False + else: + raise + +def _get_consumer(kong_admin_api_url, username): + consumers_url = "{}/consumers".format(kong_admin_api_url) + try: + response = retrying_urlopen(consumers_url + "/" + username) + consumer = json.loads(response.read()) + return consumer + except urllib2.HTTPError as e: + if(e.code == 404): + return None + else: + raise + +def _dict_without_keys(a_dict, keys): + return dict((key, a_dict[key]) for key in a_dict if key not in keys) + +def _ensure_consumer_exists(kong_admin_api_url, consumer): + username = consumer['username'] + consumers_url = "{}/consumers".format(kong_admin_api_url) + if(not _consumer_exists(kong_admin_api_url, username)): + print("Adding consumer {}".format(username)); + json_request("POST", consumers_url, {'username': username}) + + +def save_consumers(kong_admin_api_url, consumers): + consumers_url = "{}/consumers".format(kong_admin_api_url) + consumers_to_be_present = [consumer for consumer in consumers if consumer['state'] == 'present'] + consumers_to_be_absent = [consumer for consumer in consumers if consumer['state'] == 'absent'] + + for consumer in consumers_to_be_absent: + username = consumer['username'] + if(_consumer_exists(kong_admin_api_url, username)): + print("Deleting consumer {}".format(username)); + json_request("DELETE", consumers_url + "/" + username, "") + + for consumer in consumers_to_be_present: + username = consumer['username'] + _ensure_consumer_exists(kong_admin_api_url, consumer) + _save_groups_for_consumer(kong_admin_api_url, consumer) + jwt_credential = _get_first_or_create_jwt_credential(kong_admin_api_url, consumer) + credential_algorithm = jwt_credential['algorithm'] + if credential_algorithm == 'HS256': + jwt_token = jwt.encode({'iss': jwt_credential['key']}, jwt_credential['secret'], algorithm=credential_algorithm) + print("JWT token for {} is : {}".format(username, jwt_token)) + if 'print_credentials' in consumer: + print("Credentials for consumer {}, key: {}, secret: {}".format(username, jwt_credential['key'], jwt_credential['secret'])) + + saved_consumer = _get_consumer(kong_admin_api_url, username) + rate_limits = consumer.get('rate_limits') + if(rate_limits is not None): + _save_rate_limits(kong_admin_api_url, saved_consumer, rate_limits) + +def _save_rate_limits(kong_admin_api_url, saved_consumer, rate_limits): + plugin_name = 'rate-limiting' + consumer_id = saved_consumer['id'] + consumer_username = saved_consumer['username'] + for rate_limit in rate_limits: + api_name = rate_limit["api"] + saved_plugins = get_api_plugins(kong_admin_api_url, api_name) + rate_limit_plugins = [saved_plugin for saved_plugin in saved_plugins if saved_plugin['name'] == plugin_name] + rate_limit_plugins_for_this_consumer = [rate_limit_plugin for rate_limit_plugin in rate_limit_plugins if rate_limit_plugin.get('consumer_id') == consumer_id] + rate_limit_plugin_for_this_consumer = rate_limit_plugins_for_this_consumer[0] if rate_limit_plugins_for_this_consumer else None + + rate_limit_state = rate_limit.get('state', 'present') + api_pugins_url = kong_admin_api_url + "/apis/" + api_name + "/plugins" + if rate_limit_state == 'present': + rate_limit_plugin_data = _dict_without_keys(rate_limit, ['api', 'state']) + rate_limit_plugin_data['name'] = plugin_name + rate_limit_plugin_data['consumer_id'] = consumer_id + if not rate_limit_plugin_for_this_consumer: + print("Adding rate_limit for consumer {} for API {}".format(consumer_username, api_name)); + print("rate_limit_plugin_data: {}".format(rate_limit_plugin_data)) + json_request("POST", api_pugins_url, rate_limit_plugin_data) + + if rate_limit_plugin_for_this_consumer: + print("Updating rate_limit for consumer {} for API {}".format(consumer_username, api_name)); + json_request("PATCH", api_pugins_url + "/" + rate_limit_plugin_for_this_consumer["id"], rate_limit_plugin_data) + + elif rate_limit_state == 'absent': + if rate_limit_plugin_for_this_consumer: + print("Deleting rate_limit for consumer {} for API {}".format(consumer_username, api_name)); + json_request("DELETE", api_pugins_url + "/" + saved_plugin["id"], "") + + +def _get_first_or_create_jwt_credential(kong_admin_api_url, consumer): + username = consumer["username"] + credential_algorithm = consumer.get('credential_algorithm', 'HS256') + consumer_jwt_credentials_url = kong_admin_api_url + "/consumers/" + username + "/jwt" + saved_credentials_details = json.loads(retrying_urlopen(consumer_jwt_credentials_url).read()) + saved_credentials = saved_credentials_details["data"] + saved_credentials_for_algorithm = [saved_credential for saved_credential in saved_credentials if saved_credential['algorithm'] == credential_algorithm] + if(len(saved_credentials_for_algorithm) > 0): + print("Updating credentials for consumer {} for algorithm {}".format(username, credential_algorithm)); + this_credential = saved_credentials_for_algorithm[0] + credential_data = { + "rsa_public_key": consumer.get('credential_rsa_public_key', this_credential.get("rsa_public_key", '')), + "key": consumer.get('credential_iss', this_credential['key']) + } + this_credential_url = "{}/{}".format(consumer_jwt_credentials_url, this_credential["id"]) + response = json_request("PATCH", this_credential_url, credential_data) + jwt_credential = json.loads(response.read()) + return jwt_credential + else: + print("Creating jwt credentials for consumer {}".format(username)); + credential_data = { + "algorithm": credential_algorithm, + } + if 'credential_rsa_public_key' in consumer: + credential_data["rsa_public_key"] = consumer['credential_rsa_public_key'] + if 'credential_iss' in consumer: + credential_data["key"] = consumer['credential_iss'] + response = json_request("POST", consumer_jwt_credentials_url, credential_data) + jwt_credential = json.loads(response.read()) + return jwt_credential + +def _save_groups_for_consumer(kong_admin_api_url, consumer): + username = consumer["username"] + input_groups = consumer["groups"] + consumer_acls_url = kong_admin_api_url + "/consumers/" + username + "/acls" + saved_acls_details = json.loads(retrying_urlopen(consumer_acls_url).read()) + saved_acls = saved_acls_details["data"] + saved_groups = [acl["group"] for acl in saved_acls] + print("Existing groups for consumer {} : {}".format(username, saved_groups)) + print("Required groups for consumer {} : {}".format(username, input_groups)) + input_groups_to_be_created = [input_group for input_group in input_groups if input_group not in saved_groups] + saved_groups_to_be_deleted = [saved_group for saved_group in saved_groups if saved_group not in input_groups] + + for input_group in input_groups_to_be_created: + print("Adding group {} for consumer {}".format(input_group, username)); + json_request("POST", consumer_acls_url, {'group': input_group}) + + for saved_group in saved_groups_to_be_deleted: + print("Deleting group {} for consumer {}".format(saved_group, username)); + json_request("DELETE", consumer_acls_url + "/" + saved_group, "") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Configure kong consumers') + parser.add_argument('consumers_file_path', help='Path of the json file containing consumer data') + parser.add_argument('--kong-admin-api-url', help='Admin url for kong', default='http://localhost:8001') + args = parser.parse_args() + with open(args.consumers_file_path) as consumers_file: + input_consumers = json.load(consumers_file) + try: + save_consumers(args.kong_admin_api_url, input_consumers) + except urllib2.HTTPError as e: + error_message = e.read() + print(error_message) + raise diff --git a/kubernetes/helm_charts/cattle-system/rancher.yaml b/kubernetes/helm_charts/cattle-system/rancher.yaml new file mode 100644 index 000000000..fda1860f7 --- /dev/null +++ b/kubernetes/helm_charts/cattle-system/rancher.yaml @@ -0,0 +1,76 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cattle-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rancher + namespace: cattle-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + # "namespace" omitted since ClusterRoles are not namespaced + name: rancher +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +# This role binding allows "jane" to read pods in the "default" namespace. +kind: ClusterRoleBinding +metadata: + name: rancher +subjects: +- kind: ServiceAccount + name: rancher + namespace: cattle-system + apiGroup: "" +roleRef: + kind: ClusterRole #this must be Role or ClusterRole + name: rancher + apiGroup: "" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rancher + namespace: cattle-system +spec: + replicas: 1 + selector: + matchLabels: + app: rancher + template: + metadata: + labels: + app: rancher + spec: + serviceAccountName: rancher + containers: + - name: name + image: rancher/rancher:v2.3.1 + resources: + requests: + cpu: 100m + memory: 200Mi + ports: + - containerPort: 443 +--- +apiVersion: v1 +kind: Service +metadata: + name: rancher + namespace: cattle-system +spec: + type: LoadBalancer + ports: + - port: 8443 + targetPort: 443 + selector: + app: rancher diff --git a/kubernetes/helm_charts/core/adminutils/Chart.yaml b/kubernetes/helm_charts/core/adminutils/Chart.yaml new file mode 100644 index 000000000..29e0228e6 --- /dev/null +++ b/kubernetes/helm_charts/core/adminutils/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: adminutils +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/adminutils/templates/configmap.yaml b/kubernetes/helm_charts/core/adminutils/templates/configmap.yaml new file mode 100644 index 000000000..0d2fd83ac --- /dev/null +++ b/kubernetes/helm_charts/core/adminutils/templates/configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +data: + {{- range $key, $val := .Values.adminutilenv }} + {{ $key }}: {{ $val }} + {{- end }} +kind: ConfigMap +metadata: + creationTimestamp: null + name: {{ .Chart.Name }}-config + namespace: {{ .Values.namespace }} + +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: private-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/adminutils/templates/deployment.yaml b/kubernetes/helm_charts/core/adminutils/templates/deployment.yaml new file mode 100644 index 000000000..ddda16139 --- /dev/null +++ b/kubernetes/helm_charts/core/adminutils/templates/deployment.yaml @@ -0,0 +1,68 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + volumeMounts: + - mountPath: /keys/ + name: rsa-keys + volumes: + - name: rsa-keys + secret: + secretName: private-keys +{{ end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: adminutil + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/adminutils/values.j2 b/kubernetes/helm_charts/core/adminutils/values.j2 new file mode 100644 index 000000000..0e2f0bdab --- /dev/null +++ b/kubernetes/helm_charts/core/adminutils/values.j2 @@ -0,0 +1,57 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{adminutil_replicacount|default(1)}} +repository: {{adminutil_repository|default('adminutil')}} +image_tag: {{image_tag}} +resources: + requests: + cpu: {{adminutil_cpu_req|default('100m')}} + memory: {{adminutil_mem_req|default('100Mi')}} + limits: + cpu: {{adminutil_cpu_limit|default('1000m')}} + memory: {{adminutil_memory_limit|default('1024Mi')}} +network: + port: 4000 + targetport: 4000 +strategy: + type: RollingUpdate + maxsurge: {{ adminutil_maxsurge|default('25%') }} + maxunavailable: {{ adminutil_maxunavailable|default('25%') }} + +{{ adminutils_liveness_readiness | to_nice_yaml }} + +adminutilenv: + JAVA_OPTS: "-Xms{{adminutil__initial_heap_size|default('256m')}} -Xmx{{adminutil__max_heap_size|default('256m')}}" + SERVER_PORT: '"{{adminutil__port|default('4000')}}"' + AM_ADMIN_API_ENDPOINT: "http://kong.{{namespace}}.svc.cluster.local:8001" + SPRING_PROFILES_ACTIVE: {{adminutil__spring_profile|default('production')}} + DEFAULT_CONSUMER_GROUP: {{adminutil__default_consumer_group|default('contentUser')}} + ENDPOINTS_HEALTH_ID: {{adminutil__health_id|default('apihealth')}} + ENDPOINTS_HEALTH_SENSITIVE: '"{{adminutil__is_health_sensitive|default('false')}}"' + ENDPOINTS_METRICS_ID: {{adminutil__metrics_id|default('metrics')}} + ENDPOINTS_METRICS_SENSITIVE: '"{{adminutil__is_metrics_sensitive|default('false')}}"' + AM_ADMIN_API_KEYS: "mobile_device,desktop_device,access" + AM_ADMIN_API_MOBILE_DEVICE_BASEPATH: "/keys/" + AM_ADMIN_API_MOBILE_DEVICE_KEYPREFIX: '"{{ adminutil__device_keyprefix }}"' + AM_ADMIN_API_MOBILE_DEVICE_KEYSTART: '"{{ adminutil__device_keystart }}"' + AM_ADMIN_API_MOBILE_DEVICE_KEYCOUNT: '"{{ adminutil__device_keycount }}"' + AM_ADMIN_API_DESKTOP_DEVICE_BASEPATH: "/keys/" + AM_ADMIN_API_DESKTOP_DEVICE_KEYPREFIX: '"{{ adminutil__desktop_keyprefix }}"' + AM_ADMIN_API_DESKTOP_DEVICE_KEYSTART: '"{{ adminutil__desktop_keystart }}"' + AM_ADMIN_API_DESKTOP_DEVICE_KEYCOUNT: '"{{ adminutil__desktop_keycount }}"' + AM_ADMIN_API_ACCESS_BASEPATH: "/keys/" + AM_ADMIN_API_ACCESS_KEYPREFIX: '"{{ adminutil__access_keyprefix }}"' + AM_ADMIN_API_ACCESS_KEYSTART: '"{{ adminutil__access_keystart }}"' + AM_ADMIN_API_ACCESS_KEYCOUNT: '"{{ adminutil__access_keycount }}"' + REFRESH_TOKEN_PUBLIC_BASEPATH: "/keys/" + REFRESH_TOKEN_PUBLIC_KEYPREFIX: '"{{ adminutil_refresh_token_public_key_prefix }}"' + REFRESH_TOKEN_KID: '"{{ adminutil_refresh_token_public_key_kid }}"' + REFRESH_TOKEN_DOMAIN: '"{{ keycloak_auth_server_url }}/realms/{{ keycloak_realm }}"' + REFRESH_TOKEN_PRELOAD: '"{{ adminutil_refresh_token_preload }}"' + ACCESS_TOKEN_VALIDITY: '"{{ adminutil_access_token_validity }}"' + REFRESH_TOKEN_OFFLINE_VALIDITY: '"{{ adminutil_refresh_token_offline_validity }}"' + REFRESH_TOKEN_LOG_OLDER_THAN: '"{{ adminutil_refresh_token_log_older_than }}"' diff --git a/kubernetes/helm_charts/core/analytics/Chart.yaml b/kubernetes/helm_charts/core/analytics/Chart.yaml new file mode 100644 index 000000000..fad12b63f --- /dev/null +++ b/kubernetes/helm_charts/core/analytics/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: analytics +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/analytics/templates/deployment.yaml b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml new file mode 100644 index 000000000..54c4f9ab1 --- /dev/null +++ b/kubernetes/helm_charts/core/analytics/templates/deployment.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + - name: MIN_HEAP + value: {{ .Values.env.min_heap | quote }} + - name: MAX_HEAP + value: {{ .Values.env.max_heap | quote }} + - name: azure_storage_secret + value: {{ .Values.env.azure_private_account_secret | quote }} + - name: azure_storage_key + value: {{ .Values.env.azure_private_account_name | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /home/analytics/application.conf + subPath: analytics_api_service.conf + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/analytics/templates/serviceMonitor.yaml b/kubernetes/helm_charts/core/analytics/templates/serviceMonitor.yaml new file mode 100644 index 000000000..16af0de8d --- /dev/null +++ b/kubernetes/helm_charts/core/analytics/templates/serviceMonitor.yaml @@ -0,0 +1,26 @@ +{{- if .Values.serviceMonitor.enabled }} +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} +spec: + endpoints: + - honorLabels: true + interval: 60s + path: /metrics/druid + port: http-{{ .Chart.Name }} + scheme: http + scrapeTimeout: 10s + jobLabel: {{ .Chart.Name }} + namespaceSelector: + matchNames: + - {{ default .Values.namespace .Release.Namespace }} + selector: + matchLabels: + app: {{ .Chart.Name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/analytics/values.j2 b/kubernetes/helm_charts/core/analytics/values.j2 new file mode 100644 index 000000000..c1df8545b --- /dev/null +++ b/kubernetes/helm_charts/core/analytics/values.j2 @@ -0,0 +1,38 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{analytics_java_mem_limit|default('-Xmx600m')}} + min_heap: {{analytics_min_heap_limit|default('-Xms1g')}} + max_heap: {{analytics_max_heap_limit|default('-Xmx2g')}} + azure_private_account_secret: {{ sunbird_private_storage_account_key }} + azure_private_account_name: {{ sunbird_private_storage_account_name }} + +replicaCount: {{analytics_replicacount|default(1)}} +repository: {{analytics_repository|default('sunbird-analytics-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{analytics_cpu_req|default('100m')}} + memory: {{analytics_mem_req|default('100Mi')}} + limits: + cpu: {{analytics_cpu_limit|default('1')}} + memory: {{analytics_mem_limit|default('1024Mi')}} +network: + port: 9000 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ analytics_maxsurge|default('25%') }} + maxunavailable: {{ analytics_maxunavailable|default('25%') }} + +{{ analytics_liveness_readiness | to_nice_yaml }} + +serviceMonitor: + enabled: true + labels: # labels with which the prometheus choose the serviceMonitor + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/helm_charts/core/apimanager/Chart.yaml b/kubernetes/helm_charts/core/apimanager/Chart.yaml new file mode 100755 index 000000000..33becb417 --- /dev/null +++ b/kubernetes/helm_charts/core/apimanager/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: apimanager +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/apimanager/templates/configmap.yaml b/kubernetes/helm_charts/core/apimanager/templates/configmap.yaml new file mode 100755 index 000000000..8e11318ad --- /dev/null +++ b/kubernetes/helm_charts/core/apimanager/templates/configmap.yaml @@ -0,0 +1,13 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.apimanagerenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +# {{- range $key, $val := .Values.apimanagerenvOptional }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/apimanager/templates/deployment.yaml b/kubernetes/helm_charts/core/apimanager/templates/deployment.yaml new file mode 100755 index 000000000..ad1977b9b --- /dev/null +++ b/kubernetes/helm_charts/core/apimanager/templates/deployment.yaml @@ -0,0 +1,73 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + name: network + - containerPort: {{ .Values.service.port }} + name: service + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + lifecycle: + preStop: + exec: + command: + - kong + - quit + +--- +apiVersion: v1 +kind: Service +metadata: + name: kong + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.port }} + targetPort: {{ .Values.network.targetport }} + - name: http-admin-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.service.port }} + targetPort: {{ .Values.service.targetport }} + selector: + app: {{ .Chart.Name }} # metadataname of deployment diff --git a/kubernetes/helm_charts/core/apimanager/values.j2 b/kubernetes/helm_charts/core/apimanager/values.j2 new file mode 100755 index 000000000..4270bc727 --- /dev/null +++ b/kubernetes/helm_charts/core/apimanager/values.j2 @@ -0,0 +1,28 @@ +### variables for apimanager-kong service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{api_manager_replicacount|default(1)}} +repository: {{api_manager_repository|default('kong')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{api_manager_cpu_req|default('100m')}} + memory: {{api_manager_mem_req|default('100Mi')}} + limits: + cpu: {{api_manager_cpu_limit|default('1000m')}} + memory: {{api_manager_memory_limit|default('1024Mi')}} +network: + port: 8000 + targetport: 8000 +service: + port: 8001 + targetport: 8001 +strategy: + type: RollingUpdate + maxsurge: {{ apimanager_maxsurge|default('25%') }} + maxunavailable: {{ apimanager_maxunavailable|default('25%') }} + +{{ apimanager_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/apimanagerecho/Chart.yaml b/kubernetes/helm_charts/core/apimanagerecho/Chart.yaml new file mode 100755 index 000000000..34225b938 --- /dev/null +++ b/kubernetes/helm_charts/core/apimanagerecho/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: apimanagerecho +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/apimanagerecho/templates/deployment.yaml b/kubernetes/helm_charts/core/apimanagerecho/templates/deployment.yaml new file mode 100644 index 000000000..b6bc5903c --- /dev/null +++ b/kubernetes/helm_charts/core/apimanagerecho/templates/deployment.yaml @@ -0,0 +1,53 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.strategy.type }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: echo + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/apimanagerecho/values.j2 b/kubernetes/helm_charts/core/apimanagerecho/values.j2 new file mode 100755 index 000000000..ff53d1679 --- /dev/null +++ b/kubernetes/helm_charts/core/apimanagerecho/values.j2 @@ -0,0 +1,23 @@ +### variables for apimanager-kong service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{apimanagerecho_replicacount|default(1)}} +repository: {{apimanagerecho_repository|default('echo-server')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{api_managerecho_cpu_req|default('50m')}} + memory: {{api_managerecho_mem_req|default('50Mi')}} + limits: + cpu: {{api_managerecho_cpu_limit|default('500m')}} + memory: {{api_managerecho_mem_limit|default('500Mi')}} +network: + port: 9595 + targetport: 9595 +strategy: + type: RollingUpdate + +{{ api_managerecho_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/assessment/.helmignore b/kubernetes/helm_charts/core/assessment/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/assessment/Chart.yaml b/kubernetes/helm_charts/core/assessment/Chart.yaml new file mode 100644 index 000000000..a0174b78e --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: assessment +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/assessment/templates/_helpers.tpl b/kubernetes/helm_charts/core/assessment/templates/_helpers.tpl new file mode 100644 index 000000000..ad9be7363 --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "assessment.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "assessment.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "assessment.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "assessment.labels" -}} +app.kubernetes.io/name: {{ include "assessment.name" . }} +helm.sh/chart: {{ include "assessment.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/assessment/templates/configmap.yaml b/kubernetes/helm_charts/core/assessment/templates/configmap.yaml new file mode 100644 index 000000000..1f222e4f9 --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.assessmentenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/assessment/templates/deployment.yaml b/kubernetes/helm_charts/core/assessment/templates/deployment.yaml new file mode 100644 index 000000000..e805dc76d --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/templates/deployment.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /home/sunbird/assessment-service-1.0-SNAPSHOT/config/application.conf + subPath: assessment-service_application.conf + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/assessment-service-1.0-SNAPSHOT/config/logback.xml + subPath: assessment-service_logback.xml + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/assessment/values.j2 b/kubernetes/helm_charts/core/assessment/values.j2 new file mode 100644 index 000000000..d4dc22f04 --- /dev/null +++ b/kubernetes/helm_charts/core/assessment/values.j2 @@ -0,0 +1,30 @@ +#jinja2:lstrip_blocks: True + +### Default variable file for assessment-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{assesment_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{assessment_replicacount|default(1)}} +repository: {{assessment_repository|default('assessment-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{assessment_cpu_req|default('50m')}} + memory: {{assessment_mem_req|default('50Mi')}} + limits: + cpu: {{assessment_cpu_limit|default('1')}} + memory: {{assessment_mem_limit|default('500Mi')}} +network: + port: 9003 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ assessment_maxsurge|default('25%') }} + maxunavailable: {{ assessment_maxunavailable|default('25%') }} + +{{ assessment_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/badger/Chart.yaml b/kubernetes/helm_charts/core/badger/Chart.yaml new file mode 100755 index 000000000..c87735cf9 --- /dev/null +++ b/kubernetes/helm_charts/core/badger/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: badger +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/badger/templates/deployment.yaml b/kubernetes/helm_charts/core/badger/templates/deployment.yaml new file mode 100755 index 000000000..b3fbc36ca --- /dev/null +++ b/kubernetes/helm_charts/core/badger/templates/deployment.yaml @@ -0,0 +1,63 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /badger/code/apps/mainsite/settings_local.py + subPath: settings_local.py + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/badger/values.j2 b/kubernetes/helm_charts/core/badger/values.j2 new file mode 100755 index 000000000..f98f76652 --- /dev/null +++ b/kubernetes/helm_charts/core/badger/values.j2 @@ -0,0 +1,25 @@ +### Default variable file for badger-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{badger_replicacount|default(1)}} +repository: {{badger_repository|default('badger')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{badger_cpu_req|default('100m')}} + memory: {{badger_mem_req|default('100Mi')}} + limits: + cpu: {{badger_cpu_limit|default('500m')}} + memory: {{badger_mem_limit|default('500Mi')}} +network: + port: 8004 + targetport: 8004 +strategy: + type: RollingUpdate + maxsurge: {{ badger_maxsurge|default('25%') }} + maxunavailable: {{ badger_maxunavailable|default('25%') }} + +{{ badger_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/cert/Chart.yaml b/kubernetes/helm_charts/core/cert/Chart.yaml new file mode 100644 index 000000000..06916f3b5 --- /dev/null +++ b/kubernetes/helm_charts/core/cert/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: cert +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/cert/templates/configmap.yaml b/kubernetes/helm_charts/core/cert/templates/configmap.yaml new file mode 100644 index 000000000..2a18b4516 --- /dev/null +++ b/kubernetes/helm_charts/core/cert/templates/configmap.yaml @@ -0,0 +1,11 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.certenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} + diff --git a/kubernetes/helm_charts/core/cert/templates/deployment.yaml b/kubernetes/helm_charts/core/cert/templates/deployment.yaml new file mode 100644 index 000000000..0d1e794d0 --- /dev/null +++ b/kubernetes/helm_charts/core/cert/templates/deployment.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/cert/values.j2 b/kubernetes/helm_charts/core/cert/values.j2 new file mode 100644 index 000000000..b6e4488b8 --- /dev/null +++ b/kubernetes/helm_charts/core/cert/values.j2 @@ -0,0 +1,28 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{cert_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{cert_replicacount|default(1)}} +repository: {{cert_repository|default('cert_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{cert_cpu_req|default('100m')}} + memory: {{cert_mem_req|default('100Mi')}} + limits: + cpu: {{cert_cpu_limit|default('1')}} + memory: {{cert_mem_limit|default('1024Mi')}} +network: + port: 9011 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ cert_maxsurge|default('25%') }} + maxunavailable: {{ cert_maxunavailable|default('25%') }} + +{{ cert_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/certregistry/Chart.yaml b/kubernetes/helm_charts/core/certregistry/Chart.yaml new file mode 100644 index 000000000..7e55b46ee --- /dev/null +++ b/kubernetes/helm_charts/core/certregistry/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: certregistry +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/certregistry/templates/configmap.yaml b/kubernetes/helm_charts/core/certregistry/templates/configmap.yaml new file mode 100644 index 000000000..d33ab5474 --- /dev/null +++ b/kubernetes/helm_charts/core/certregistry/templates/configmap.yaml @@ -0,0 +1,11 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.certregistryenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} + diff --git a/kubernetes/helm_charts/core/certregistry/templates/deployment.yaml b/kubernetes/helm_charts/core/certregistry/templates/deployment.yaml new file mode 100644 index 000000000..bcfa950e3 --- /dev/null +++ b/kubernetes/helm_charts/core/certregistry/templates/deployment.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: cert-registry-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/certregistry/values.j2 b/kubernetes/helm_charts/core/certregistry/values.j2 new file mode 100644 index 000000000..9ba6889f7 --- /dev/null +++ b/kubernetes/helm_charts/core/certregistry/values.j2 @@ -0,0 +1,28 @@ +### Default variable file for cert-registry-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{certregistry_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{certregistry_replicacount|default(1)}} +repository: {{certregistry_repository|default('cert_registry_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{certregistry_cpu_req|default('100m')}} + memory: {{certregistry_mem_req|default('100Mi')}} + limits: + cpu: {{certregistry_cpu_limit|default('1')}} + memory: {{certregistry_mem_limit|default('1024Mi')}} +network: + port: 9013 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ certregistry_maxsurge|default('25%') }} + maxunavailable: {{ certregistry_maxunavailable|default('25%') }} + +{{ cert_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/content/Chart.yaml b/kubernetes/helm_charts/core/content/Chart.yaml new file mode 100644 index 000000000..dbf9d485c --- /dev/null +++ b/kubernetes/helm_charts/core/content/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: content +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/content/templates/deployment.yaml b/kubernetes/helm_charts/core/content/templates/deployment.yaml new file mode 100644 index 000000000..714521a20 --- /dev/null +++ b/kubernetes/helm_charts/core/content/templates/deployment.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /home/sunbird/content-service-1.0-SNAPSHOT/config/application.conf + subPath: content-service_application.conf + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/content-service-1.0-SNAPSHOT/config/logback.xml + subPath: content-service_logback.xml + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/content/values.j2 b/kubernetes/helm_charts/core/content/values.j2 new file mode 100644 index 000000000..b2d93411b --- /dev/null +++ b/kubernetes/helm_charts/core/content/values.j2 @@ -0,0 +1,28 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{content_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{content_replicacount|default(1)}} +repository: {{content_repository|default('content-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{content_cpu_req|default('100m')}} + memory: {{content_mem_req|default('100Mi')}} + limits: + cpu: {{content_cpu_limit|default('1')}} + memory: {{content_mem_limit|default('1024Mi')}} +network: + port: 9002 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ content_maxsurge|default('25%') }} + maxunavailable: {{ content_maxunavailable|default('25%') }} + +{{ content_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/discussionsmw/.helmignore b/kubernetes/helm_charts/core/discussionsmw/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/discussionsmw/Chart.yaml b/kubernetes/helm_charts/core/discussionsmw/Chart.yaml new file mode 100644 index 000000000..bdb14d7af --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: discussionsmw +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/discussionsmw/templates/_helpers.tpl b/kubernetes/helm_charts/core/discussionsmw/templates/_helpers.tpl new file mode 100644 index 000000000..4dda28416 --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "content.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "content.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "content.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "content.labels" -}} +app.kubernetes.io/name: {{ include "content.name" . }} +helm.sh/chart: {{ include "content.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/discussionsmw/templates/configmap.yaml b/kubernetes/helm_charts/core/discussionsmw/templates/configmap.yaml new file mode 100644 index 000000000..e86595441 --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/templates/configmap.yaml @@ -0,0 +1,11 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.knowledgemwenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +# sunbird_cassandra_replication_strategy: {{ .Values.sunbird_cassandra_replication_strategy | toJson }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/discussionsmw/templates/deployment.yaml b/kubernetes/helm_charts/core/discussionsmw/templates/deployment.yaml new file mode 100644 index 000000000..7cfc8ef45 --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: discussionsmw-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: disussionsmw-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/discussionsmw/values.j2 b/kubernetes/helm_charts/core/discussionsmw/values.j2 new file mode 100644 index 000000000..220b22d58 --- /dev/null +++ b/kubernetes/helm_charts/core/discussionsmw/values.j2 @@ -0,0 +1,25 @@ +### Default variable file for disussions_mw service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{disussions_mw_replicacount|default(1)}} +repository: {{disussions_mw_repository|default('discussion-middleware')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{disussions_mw_cpu_req|default('100m')}} + memory: {{disussions_mw_mem_req|default('100Mi')}} + limits: + cpu: {{disussions_mw_cpu_limit|default('1')}} + memory: {{disussions_mw_mem_limit|default('1024Mi')}} +network: + port: 3002 + targetport: 3002 +strategy: + type: RollingUpdate + maxsurge: {{ disussions_mw_maxsurge|default('25%') }} + maxunavailable: {{ disussions_mw_maxunavailable|default('25%') }} + +{{ discussionsmw_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/enc/Chart.yaml b/kubernetes/helm_charts/core/enc/Chart.yaml new file mode 100644 index 000000000..91c8b3224 --- /dev/null +++ b/kubernetes/helm_charts/core/enc/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: enc +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/enc/templates/configmap.yaml b/kubernetes/helm_charts/core/enc/templates/configmap.yaml new file mode 100644 index 000000000..1d5908d5e --- /dev/null +++ b/kubernetes/helm_charts/core/enc/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.encenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/enc/templates/deployment.yaml b/kubernetes/helm_charts/core/enc/templates/deployment.yaml new file mode 100644 index 000000000..731699890 --- /dev/null +++ b/kubernetes/helm_charts/core/enc/templates/deployment.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/enc/values.j2 b/kubernetes/helm_charts/core/enc/values.j2 new file mode 100644 index 000000000..3f375b060 --- /dev/null +++ b/kubernetes/helm_charts/core/enc/values.j2 @@ -0,0 +1,25 @@ +### Default variable file for enc-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{enc_replicacount|default(1)}} +repository: {{enc_repository|default('enc_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{enc_cpu_req|default('100m')}} + memory: {{enc_mem_req|default('100Mi')}} + limits: + cpu: {{enc_cpu_limit|default('1')}} + memory: {{enc_mem_limit|default('1024Mi')}} +network: + port: 9010 + targetport: 8013 +strategy: + type: RollingUpdate + maxsurge: {{ enc_maxsurge|default('25%') }} + maxunavailable: {{ enc_maxunavailable|default('25%') }} + +{{ enc_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/groups/Chart.yaml b/kubernetes/helm_charts/core/groups/Chart.yaml new file mode 100644 index 000000000..abf54a22e --- /dev/null +++ b/kubernetes/helm_charts/core/groups/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: groups +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/groups/templates/configmap.yaml b/kubernetes/helm_charts/core/groups/templates/configmap.yaml new file mode 100644 index 000000000..810570ef9 --- /dev/null +++ b/kubernetes/helm_charts/core/groups/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: groups-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/groups/templates/deployment.yaml b/kubernetes/helm_charts/core/groups/templates/deployment.yaml new file mode 100644 index 000000000..3f12a3346 --- /dev/null +++ b/kubernetes/helm_charts/core/groups/templates/deployment.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + volumeMounts: + - mountPath: {{ .Values.groups_device_basepath }} + name: access-keys + volumes: + - name: access-keys + secret: + secretName: groups-access-keys +{{ end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/groups/values.j2 b/kubernetes/helm_charts/core/groups/values.j2 new file mode 100644 index 000000000..43f1928a6 --- /dev/null +++ b/kubernetes/helm_charts/core/groups/values.j2 @@ -0,0 +1,31 @@ +### Default variable file for groups-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + + +env: + javaoptions: {{groups_java_mem_limit|default('-Xmx500m')}} + +replicaCount: {{groups_replicacount|default(1)}} +repository: {{groups_repository|default('groups_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{groups_cpu_req|default('100m')}} + memory: {{groups_mem_req|default('300Mi')}} + limits: + cpu: {{groups_cpu_limit|default('1')}} + memory: {{groups_mem_limit|default('1024Mi')}} +network: + port: 9000 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ groups_maxsurge|default('25%') }} + maxunavailable: {{ groups_maxunavailable|default('25%') }} + +{{ groups_liveness_readiness | to_nice_yaml }} + +groups_device_basepath: {{ groups_device_basepath | default('/keys/') }} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/.helmignore b/kubernetes/helm_charts/core/hawkeye-superset/.helmignore new file mode 100644 index 000000000..75840bb78 --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/.helmignore @@ -0,0 +1,18 @@ +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/kubernetes/helm_charts/core/hawkeye-superset/Chart.yaml b/kubernetes/helm_charts/core/hawkeye-superset/Chart.yaml new file mode 100644 index 000000000..5e800b8a6 --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: superset +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/NOTES.txt b/kubernetes/helm_charts/core/hawkeye-superset/templates/NOTES.txt new file mode 100644 index 000000000..ce8db252c --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/NOTES.txt @@ -0,0 +1,19 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range .Values.ingress.hosts }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}{{ $.Values.ingress.path }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ template "superset.fullname" . }}-service) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get svc -w {{ template "superset.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ template "superset.fullname" . }} -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app={{ template "superset.name" . }},release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl port-forward $POD_NAME 8080:80 +{{- end }} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/_helpers.tpl b/kubernetes/helm_charts/core/hawkeye-superset/templates/_helpers.tpl new file mode 100644 index 000000000..d92099bf1 --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "superset.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "superset.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "superset.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/configmap.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/configmap.yaml new file mode 100644 index 000000000..afb7ef517 --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: superset-configmap + labels: + app: {{ template "superset.name" . }} + chart: {{ template "superset.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: +{{ (.Files.Glob "config/*").AsConfig | indent 2 }} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/deployment.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/deployment.yaml new file mode 100644 index 000000000..207cbeb7e --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/deployment.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "superset.fullname" . }} + labels: + app: {{ template "superset.name" . }} + chart: {{ template "superset.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "superset.name" . }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "superset.name" . }} + release: {{ .Release.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + volumeMounts: + - name: superset-config + mountPath: /etc/superset/ + env: + - name: "SUPERSET_PORT" + value: {{ .Values.service.port | quote}} + ports: + - name: http + containerPort: 8088 + protocol: TCP + resources: +{{ toYaml .Values.resources | indent 12 }} + envFrom: + - configMapRef: + name: hawkeye-envconfigmap + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} + volumes: + - name: "superset-config" + configMap: + name: superset-configmap diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/heconfigmap.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/heconfigmap.yaml new file mode 100644 index 000000000..d0ab37988 --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/heconfigmap.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: hawkeye-envconfigmap +data: + {{- range $key, $val := .Values.envvar }} + {{ $key }}: {{ $val }} + {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/ingress.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/ingress.yaml new file mode 100644 index 000000000..9cbec2c1f --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/ingress.yaml @@ -0,0 +1,38 @@ +{{ if .Values.ingress.enabled -}} +{{- $fullName := include "superset.fullname" . -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "superset.name" . }} + chart: {{ template "superset.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . }} + http: + paths: + - path: {{ $ingressPath }} + backend: + serviceName: {{ $fullName }} + servicePort: http + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/init-job.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/init-job.yaml new file mode 100644 index 000000000..5b987d35b --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/init-job.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "superset.name" . }}-init-db +spec: + template: + metadata: + name: {{ template "superset.name" . }}-init-db + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + securityContext: + runAsUser: 0 # Needed in order to allow pip install to work in bootstrap + containers: + - name: {{ template "superset.name" . }}-init-db + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + env: + {{- range $key, $value := .Values.envvar }} + - name: {{ $key | quote }} + value: {{ $value | quote }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: {{ tpl (toJson .Values.init.command) . }} + restartPolicy: Never diff --git a/kubernetes/helm_charts/core/hawkeye-superset/templates/service.yaml b/kubernetes/helm_charts/core/hawkeye-superset/templates/service.yaml new file mode 100644 index 000000000..31a1571cb --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "superset.fullname" . }}-service + labels: + app: {{ template "superset.name" . }} + chart: {{ template "superset.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + nodePort: 31566 + protocol: TCP + name: http + selector: + app: {{ template "superset.name" . }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/core/hawkeye-superset/values.j2 b/kubernetes/helm_charts/core/hawkeye-superset/values.j2 new file mode 100644 index 000000000..18300b24a --- /dev/null +++ b/kubernetes/helm_charts/core/hawkeye-superset/values.j2 @@ -0,0 +1,59 @@ +replicaCount: 1 + +image: + repository: {{ dockerhub }}/hawkeye + tag: {{ image_tag }} + pullPolicy: Always + +imagepullsecrets: {{ imagepullsecrets }} + +service: + type: NodePort + port: 8088 + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + hosts: + - chart-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +envvar: + PG_USER: "{{ dp_postgres_username }}" + PG_PASS: "{{ dp_postgres_password }}" + PG_HOST: "{{ dp_postgres_host }}" + PORTAL_HOST: "{{ proto }}://{{ domain_name }}" + PORTAL_API_HOST: "{{ proto }}://{{ domain_name }}/api/data/v1/report-service" + PORTAL_API_KEY: {{ core_vault_sunbird_api_auth_token }} + ANALYTICS_API_KEY: {{ core_vault_sunbird_api_auth_token }} + ANALYTICS_API_HOST: http://{{analyticsapi_ip}}:9000 + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +init: + command: + - "/bin/sh" + - "-c" + - "superset db upgrade && superset init && superset fab create-admin --username admin --firstname Superset --lastname Admin --email admin@superset.com --password admin" diff --git a/kubernetes/helm_charts/core/knowledgemw/.helmignore b/kubernetes/helm_charts/core/knowledgemw/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/knowledgemw/Chart.yaml b/kubernetes/helm_charts/core/knowledgemw/Chart.yaml new file mode 100644 index 000000000..c9f1768e6 --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: knowledgemw +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/knowledgemw/templates/_helpers.tpl b/kubernetes/helm_charts/core/knowledgemw/templates/_helpers.tpl new file mode 100644 index 000000000..4dda28416 --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "content.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "content.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "content.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "content.labels" -}} +app.kubernetes.io/name: {{ include "content.name" . }} +helm.sh/chart: {{ include "content.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/knowledgemw/templates/configmap.yaml b/kubernetes/helm_charts/core/knowledgemw/templates/configmap.yaml new file mode 100644 index 000000000..eb2c944f4 --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/templates/configmap.yaml @@ -0,0 +1,11 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.knowledgemwenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +# sunbird_cassandra_replication_strategy: {{ .Values.sunbird_cassandra_replication_strategy | toJson }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} \ No newline at end of file diff --git a/kubernetes/helm_charts/core/knowledgemw/templates/deployment.yaml b/kubernetes/helm_charts/core/knowledgemw/templates/deployment.yaml new file mode 100644 index 000000000..78442e62a --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: knowledgemw-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: knowledge-mw-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/knowledgemw/values.j2 b/kubernetes/helm_charts/core/knowledgemw/values.j2 new file mode 100644 index 000000000..bf6ae52c1 --- /dev/null +++ b/kubernetes/helm_charts/core/knowledgemw/values.j2 @@ -0,0 +1,25 @@ +### Default variable file for knowledge_mw service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{knowledge_mw_replicacount|default(1)}} +repository: {{knowledge_mw_repository|default('knowledge-mw-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{knowledge_mw_cpu_req|default('100m')}} + memory: {{knowledge_mw_mem_req|default('100Mi')}} + limits: + cpu: {{knowledge_mw_cpu_limit|default('1')}} + memory: {{knowledge_mw_mem_limit|default('1024Mi')}} +network: + port: 5000 + targetport: 5000 +strategy: + type: RollingUpdate + maxsurge: {{ knowledge_mw_maxsurge|default('25%') }} + maxunavailable: {{ knowledge_mw_maxunavailable|default('25%') }} + +{{ knowledgemw_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/learner/.helmignore b/kubernetes/helm_charts/core/learner/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/learner/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/learner/Chart.yaml b/kubernetes/helm_charts/core/learner/Chart.yaml new file mode 100644 index 000000000..d2c54a936 --- /dev/null +++ b/kubernetes/helm_charts/core/learner/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: learner +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/learner/templates/_helpers.tpl b/kubernetes/helm_charts/core/learner/templates/_helpers.tpl new file mode 100644 index 000000000..420ba7943 --- /dev/null +++ b/kubernetes/helm_charts/core/learner/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "learner.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "learner.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "learner.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "learner.labels" -}} +app.kubernetes.io/name: {{ include "learner.name" . }} +helm.sh/chart: {{ include "learner.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/learner/templates/configmap.yaml b/kubernetes/helm_charts/core/learner/templates/configmap.yaml new file mode 100644 index 000000000..f6dacc41b --- /dev/null +++ b/kubernetes/helm_charts/core/learner/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: learner-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/learner/templates/deployment.yaml b/kubernetes/helm_charts/core/learner/templates/deployment.yaml new file mode 100644 index 000000000..81117a28f --- /dev/null +++ b/kubernetes/helm_charts/core/learner/templates/deployment.yaml @@ -0,0 +1,80 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/learner/learning-service-1.0-SNAPSHOT/config/logback.xml + subPath: learner-service_logback.xml +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - mountPath: {{ .Values.learner_device_basepath }} + name: access-keys +{{- end }} + volumes: + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - name: access-keys + secret: + secretName: learner-access-keys +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/learner/values.j2 b/kubernetes/helm_charts/core/learner/values.j2 new file mode 100644 index 000000000..0faa781ea --- /dev/null +++ b/kubernetes/helm_charts/core/learner/values.j2 @@ -0,0 +1,31 @@ + +### Default variable file for learner-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{learner_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{learner_replicacount|default(1)}} +repository: {{learner_repository|default('learner_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{learner_cpu_req|default('100m')}} + memory: {{learner_mem_req|default('100Mi')}} + limits: + cpu: {{learner_cpu_limit|default('1')}} + memory: {{learner_mem_limit|default('1024Mi')}} +network: + port: 9000 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ learner_maxsurge|default('25%') }} + maxunavailable: {{ learner_maxunavailable|default('25%') }} + +{{ learner_liveness_readiness | to_nice_yaml }} + +learner_device_basepath: {{ learner_device_basepath | default('/keys/') }} diff --git a/kubernetes/helm_charts/core/lms/Chart.yaml b/kubernetes/helm_charts/core/lms/Chart.yaml new file mode 100644 index 000000000..5ebad8066 --- /dev/null +++ b/kubernetes/helm_charts/core/lms/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: lms +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/lms/templates/configmap.yaml b/kubernetes/helm_charts/core/lms/templates/configmap.yaml new file mode 100644 index 000000000..07d61ad08 --- /dev/null +++ b/kubernetes/helm_charts/core/lms/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: lms-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/lms/templates/deployment.yaml b/kubernetes/helm_charts/core/lms/templates/deployment.yaml new file mode 100644 index 000000000..1c797c37e --- /dev/null +++ b/kubernetes/helm_charts/core/lms/templates/deployment.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/lms/lms-service-1.0-SNAPSHOT/config/logback.xml + subPath: lms-service_logback.xml +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - mountPath: {{ .Values.lms_device_basepath }} + name: access-keys +{{- end }} + volumes: + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - name: access-keys + secret: + secretName: lms-access-keys +{{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/lms/values.j2 b/kubernetes/helm_charts/core/lms/values.j2 new file mode 100644 index 000000000..0714daad9 --- /dev/null +++ b/kubernetes/helm_charts/core/lms/values.j2 @@ -0,0 +1,31 @@ +### Default variable file for lms-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + + +env: + javaoptions: {{lms_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{lms_replicacount|default(1)}} +repository: {{lms_repository|default('lms_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{lms_cpu_req|default('100m')}} + memory: {{lms_mem_req|default('100Mi')}} + limits: + cpu: {{lms_cpu_limit|default('1')}} + memory: {{lms_mem_limit|default('1024Mi')}} +network: + port: 9005 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ lms_maxsurge|default('25%') }} + maxunavailable: {{ lms_maxunavailable|default('25%') }} + +{{ lms_liveness_readiness | to_nice_yaml }} + +lms_device_basepath: {{ lms_device_basepath | default('/keys/') }} diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/.helmignore b/kubernetes/helm_charts/core/nginx-private-ingress/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/Chart.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/Chart.yaml new file mode 100644 index 000000000..5d95ebd90 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: nginx-private-ingress +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.16.0 diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/_helpers.tpl b/kubernetes/helm_charts/core/nginx-private-ingress/templates/_helpers.tpl new file mode 100644 index 000000000..e910d49f3 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "private-ingress.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "private-ingress.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "private-ingress.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "private-ingress.labels" -}} +helm.sh/chart: {{ include "private-ingress.chart" . }} +{{ include "private-ingress.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "private-ingress.selectorLabels" -}} +app.kubernetes.io/name: {{ include "private-ingress.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "private-ingress.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "private-ingress.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml new file mode 100644 index 000000000..a79679825 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -0,0 +1,149 @@ +apiVersion: v1 +data: + nginx.conf: | + user nginx; + worker_processes 1; + + error_log /var/log/nginx/error.log warn; + pid /var/run/nginx.pid; + + events { + worker_connections 1024; + } + + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + lua_load_resty_core off; + lua_package_path "/etc/nginx/lua_modules/?.lua"; + log_format main '$remote_addr - $remote_user [$time_local] ' + '"$request" $status $request_length $body_bytes_sent' + ' $request_time $upstream_response_time $pipe' + ' "$http_referer" "$http_user_agent" "$sb_request_id"' + ' "$http_x_device_id" "$http_x_channel_id" "$http_x_app_id"' + ' "$http_x_app_ver" "$http_x_session_id"'; + + # If the client send request_id it should be preffered over the default one + map $http_x_request_id $sb_request_id { + default $http_x_request_id; + '' $request_id; + } + + access_log /var/log/nginx/access.log main; + include /etc/nginx/conf.d/*.conf; + sendfile on; + client_max_body_size 60M; + + server { + listen 80; + listen [::]:80; + + resolver {{ .Values.kube_dns_ip }}; + + location /learner/ { + set $target http://learner-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/learner/(.*) /$1 break; + proxy_pass $target; + } + location /api/ { + set $target http://kong.{{ .Values.namespace }}.svc.cluster.local:8000; + rewrite ^/api/(.*) /$1 break; + proxy_pass $target; + } + location /grafana/ { + set $target http://prometheus-operator-grafana.monitoring.svc.cluster.local; + rewrite ^/grafana/(.*) /$1 break; + proxy_pass $target; + } + location /admin-api/ { + set $target http://kong.{{ .Values.namespace }}.svc.cluster.local:8001; + rewrite ^/admin-api/(.*) /$1 break; + proxy_pass $target; + } + location /player/ { + set $target http://player.{{ .Values.namespace }}.svc.cluster.local:3000; + rewrite ^/player/(.*) /$1 break; + proxy_pass $target; + } + location /knowledgemw/ { + set $target http://knowledge-mw-service.{{ .Values.namespace }}.svc.cluster.local:5000; + rewrite ^/knowledgemw/(.*) /$1 break; + proxy_pass $target; + } + location /content/ { + set $target http://content-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/content/(.*) /$1 break; + proxy_pass $target; + } + location /badger/ { + set $target http://badger-service.{{ .Values.namespace }}.svc.cluster.local:8004; + rewrite ^/badger/(.*) /$1 break; + proxy_pass $target; + } + location /cert/ { + set $target http://cert-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/cert/(.*) /$1 break; + proxy_pass $target; + } + location /certreg/ { + set $target http://cert-registry-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/certreg/(.*) /$1 break; + proxy_pass $target; + } + location /print/ { + set $target http://print-service.{{ .Values.namespace }}.svc.cluster.local:5000; + rewrite ^/print/(.*) /$1 break; + proxy_pass $target; + } + location /assessment/ { + set $target http://assessment-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/assessment/(.*) /$1 break; + proxy_pass $target; + } + location /notification/ { + set $target http://notification-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/notification/(.*) /$1 break; + proxy_pass $target; + } + location /report/ { + set $target http://report-service.{{ .Values.namespace }}.svc.cluster.local:3030; + rewrite ^/report/(.*) /$1 break; + proxy_pass $target; + } + location /search/ { + set $target http://search-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/search/(.*) /$1 break; + proxy_pass $target; + } + location /lms/ { + set $target http://lms-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/lms/(.*) /$1 break; + proxy_pass $target; + } + + location /taxonomy/ { + set $target http://taxonomy-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/taxonomy/(.*) /$1 break; + proxy_pass $target; + } + location /analytics/ { + set $target http://analytics-service.{{ .Values.namespace }}.svc.cluster.local:9000; + rewrite ^/analytics/(.*) /$1 break; + proxy_pass $target; + } + location /enc/ { + set $target http://enc-service.{{ .Values.namespace }}.svc.cluster.local:8013; + rewrite ^/enc/(.*) /$1 break; + proxy_pass $target; + } + location /nodebb/ { + set $target http://nodebb-service.{{ .Values.namespace }}.svc.cluster.local:4567; + rewrite ^/enc/(.*) /$1 break; + proxy_pass $target; + } + } + } +kind: ConfigMap +metadata: + name: nginx-private-ingress.conf + namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/deployment.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/deployment.yaml new file mode 100644 index 000000000..585058faf --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/deployment.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-private-ingress + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: nginx-private-ingress + template: + metadata: + labels: + app: nginx-private-ingress + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: nginx-private + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: 80 + volumeMounts: + - mountPath: /etc/nginx/nginx.conf + name: config-volume + subPath: nginx.conf + readOnly: true + volumes: + - name: config-volume + configMap: + name: nginx-private-ingress.conf +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-private-ingress + namespace: {{ .Values.namespace }} + annotations: +{{- if .Values.annotations }} +{{ toYaml .Values.annotations | indent 4 }} +{{- end }} +{{- if .Values.private_ingress_custom_annotations }} + {{ .Values.private_ingress_annotation.cloud_provider }} +{{- end }} +spec: + type: {{ .Values.nginx_private_ingress_type }} +{{- if and .Values.nginx_private_ingress_ip (ne .Values.nginx_private_ingress_type "NodePort") }} + loadBalancerIP: {{ .Values.nginx_private_ingress_ip }} +{{- end }} + ports: + - port: 80 + nodePort: 31480 + targetPort: 80 + name: http + selector: + app: nginx-private-ingress diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 new file mode 100644 index 000000000..64f3b5ad2 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-private-ingress/values.j2 @@ -0,0 +1,31 @@ +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} +repository: {{proxy_repository|default('proxy')}} +image_tag: {{ image_tag }} +replicaCount: {{nginx_private_ingress_replicacount|default(1)}} +nginx_private_ingress_type: {{ nginx_private_ingress_type | default('LoadBalancer') }} +private_ingress_custom_annotations: {{ private_ingress_custom_annotations | d('false') | lower }} +kube_dns_ip: {{kube_dns_ip}} + +{% if nginx_private_ingress_annotations is defined and nginx_private_ingress_annotations %} +# you can define annotations in ansible for custom behaviour +# eg: +# nginx_private_ingress_annotations: +# cloud.google.com/load-balancer-type: "Internal" + +annotations: {{nginx_private_ingress_annotations|to_json}} +{% endif %} + +private_ingress_annotation: + cloud_provider: 'service.beta.kubernetes.io/azure-load-balancer-internal: "true"' +{% if private_ingressgateway_ip is defined and private_ingressgateway_ip %} +nginx_private_ingress_ip: {{ private_ingressgateway_ip }} +{% endif %} +resources: + requests: + cpu: {{nginx_private_ingress_cpu_req | default('100m')}} + memory: {{nginx_private_ingress_mem_req | default('100Mi')}} + limits: + cpu: {{nginx_private_ingress_cpu_limit | default('500m')}} + memory: {{nginx_private_ingress_mem_limit | default('500Mi')}} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/.helmignore b/kubernetes/helm_charts/core/nginx-public-ingress/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/Chart.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/Chart.yaml new file mode 100644 index 000000000..ded713533 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/Chart.yaml @@ -0,0 +1,21 @@ +apiVersion: v2 +name: nginx-public-ingress +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. +appVersion: 1.16.0 diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/_helpers.tpl b/kubernetes/helm_charts/core/nginx-public-ingress/templates/_helpers.tpl new file mode 100644 index 000000000..0af5bc238 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/_helpers.tpl @@ -0,0 +1,63 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "nginx-public-ingress.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nginx-public-ingress.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nginx-public-ingress.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "nginx-public-ingress.labels" -}} +helm.sh/chart: {{ include "nginx-public-ingress.chart" . }} +{{ include "nginx-public-ingress.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "nginx-public-ingress.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nginx-public-ingress.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nginx-public-ingress.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "nginx-public-ingress.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml new file mode 100644 index 000000000..87f6b5716 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/configMap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: proxy-default + namespace: {{ .Values.namespace }} +data: + proxy-default.conf: | +{{ .Values.proxyconfig | indent 4 }} + compression.conf: | +{{ .Values.compressionConfig | indent 4 }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-conf + namespace: {{ .Values.namespace }} +data: + nginx.conf: | +{{ .Values.nginxconfig | indent 4 }} + +--- +{{- if .Values.merge_domain_status }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: keycloak-conf + namespace: {{ .Values.namespace }} +data: + keycloak.conf: | +{{ .Values.keycloakconf | indent 4 }} +{{- end }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml new file mode 100644 index 000000000..9fc5ce4cf --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/daemonset.yaml @@ -0,0 +1,100 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nginx-public-ingress + namespace: {{ .Values.namespace }} +spec: + selector: + matchLabels: + app: nginx-public-ingress + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 25% + template: + metadata: + annotations: + nginxRolloutID: {{ randAlphaNum 5 | quote }} # Restart nginx after every deployment + fluentbit.io/parser: nginx2 + labels: + app: nginx-public-ingress + spec: + # Running nginx with custom config +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: tls + secret: + secretName: ingress-cert + - name: proxy-config + configMap: + name: proxy-default + - name: nginx-config + configMap: + name: nginx-conf +{{- if .Values.volumes }} +{{ toYaml .Values.volumes | indent 8 }} +{{- end }} + containers: + - name: nginx-public + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + resources: +{{ toYaml .Values.resources | indent 10 }} + volumeMounts: + - name: tls + mountPath: /etc/secrets + readOnly: true + - name: proxy-config + mountPath: /etc/nginx/defaults.d + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf +{{- if .Values.volumeMounts }} +{{ toYaml .Values.volumeMounts | indent 10 }} +{{- end }} + ports: + - containerPort: 80 + name: http + - containerPort: 443 + name: https + - containerPort: 9145 + name: http-metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-public-ingress-metrics + namespace: {{ .Values.namespace }} + labels: +{{- include "nginx-public-ingress.labels" . | nindent 4 }} +spec: + selector: + app: nginx-public-ingress + ports: + - name: http + port: 9145 + targetPort: 9145 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-public-ingress + namespace: {{ .Values.namespace }} +{{- if .Values.service.annotations }} + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +{{- end }} +spec: + externalTrafficPolicy: Local + selector: + app: nginx-public-ingress + type: {{ .Values.service.type }} +{{- if and .Values.service.nginx_public_ingress_ip (ne .Values.service.type "NodePort") }} + loadBalancerIP: {{ .Values.service.nginx_public_ingress_ip }} +{{- end }} + ports: +{{ toYaml .Values.service.ports | indent 4 }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/recordingRules.yaml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/recordingRules.yaml new file mode 100644 index 000000000..1701d0833 --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/recordingRules.yaml @@ -0,0 +1,50 @@ +{{- if .Values.serviceMonitor.enabled }} +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "nginx-public-ingress.fullname" . }}-recording.rule + namespace: {{ default .Values.namespace .Release.Namespace }} + labels: + {{- include "nginx-public-ingress.labels" . | nindent 4 }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} +spec: + groups: + - name: nginx-recording.rules + rules: + - expr: sum(irate(nginx_http_requests_total[1m])) by (cache_status, host, env, status) + record: job:nginx_http_requests_total:irate:sum:1m + - expr: sum(irate(nginx_http_requests_total[5m])) by (cache_status, host, env, status) + record: job:nginx_http_requests_total:irate:sum:5m + - expr: sum(irate(nginx_http_requests_total[15m])) by (cache_status, host, env, status) + record: job:nginx_http_requests_total:irate:sum:15m + - record: job:nginx_http_success_percentage:increase:sum:5m + expr: |- + ( + sum(increase(nginx_http_requests_total{status!~"4..|5.."}[5m])) by (status) + / + ignoring(status) group_left sum(increase(nginx_http_requests_total[5m])) + ) * 100 + - record: job:nginx_http_success_percentage:increase:sum:15m + expr: |- + ( + sum(increase(nginx_http_requests_total{status!~"4..|5.."}[15m])) by (status) + / + ignoring(status) group_left sum(increase(nginx_http_requests_total[15m])) + ) * 100 + - record: job:nginx_http_failure_percentage:increase:sum:5m + expr: |- + ( + sum(increase(nginx_http_requests_total{status!~"2..|3.."}[5m])) by (status) + / + ignoring(status) group_left sum(increase(nginx_http_requests_total[5m])) + ) * 100 + - record: job:nginx_http_failure_percentage:increase:sum:15m + expr: |- + ( + sum(increase(nginx_http_requests_total{status!~"2..|3.."}[15m])) by (status) + / + ignoring(status) group_left sum(increase(nginx_http_requests_total[15m])) + ) * 100 +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/templates/serviceMonitor.yml b/kubernetes/helm_charts/core/nginx-public-ingress/templates/serviceMonitor.yml new file mode 100644 index 000000000..ea8ea50cf --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/templates/serviceMonitor.yml @@ -0,0 +1,27 @@ +{{- if .Values.serviceMonitor.enabled }} +{{- if and ( .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" ) ( .Values.serviceMonitor.enabled ) }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "nginx-public-ingress.fullname" . }} + namespace: {{ default .Values.namespace .Release.Namespace }} + labels: + {{- include "nginx-public-ingress.labels" . | nindent 4 }} + {{- toYaml .Values.serviceMonitor.labels | nindent 4 }} +spec: + endpoints: + - honorLabels: true + interval: 15s + path: /metrics + port: http + scheme: http + scrapeTimeout: 10s + jobLabel: nginx-public-ingress + namespaceSelector: + matchNames: + - {{ default .Values.namespace .Release.Namespace }} + selector: + matchLabels: + {{- include "nginx-public-ingress.labels" . | nindent 6 }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 new file mode 100644 index 000000000..10ae2a8aa --- /dev/null +++ b/kubernetes/helm_charts/core/nginx-public-ingress/values.j2 @@ -0,0 +1,1199 @@ +#jinja2:lstrip_blocks: True + +namespace: {{ namespace }} +merge_domain_status: {{ merge_domain_status | lower }} +service: + annotations: {{nginx_public_ingress_service_annotations | d('') | to_json}} + type: {{ nginx_public_ingress_type | default('LoadBalancer') }} + {% if nginx_public_ingress_ip is defined %} + nginx_public_ingress_ip: {{ nginx_public_ingress_ip }} + {% endif %} + ports: + - port: 80 + name: http + targetPort: 80 + nodePort: 31380 + - port: 443 + name: https + targetPort: 443 + nodePort: 31390 + +{% if nginx_volumes is defined and nginx_volumes %} +{# +This is for custom nginx volume mount options in common.yaml +example: +nginx_volumes: + volumes: + - name: tls + secret: + secretName: ingress-cert + - name: proxy-config + configMap: + name: proxy-default + - name: nginx-config + configMap: + name: nginx-conf + volumemounts: + - name: tls + mountPath: /etc/secrets + readOnly: true + - name: proxy-config + mountPath: /etc/nginx/conf.d/default.conf + subPath: proxy.default.conf + readOnly: true + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: true +#} +volumes: {{ nginx_volumes.volumes | to_json }} +volumeMounts: {{ nginx_volumes.volumeMounts | to_json }} +{% endif %} + +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +resources: + requests: + cpu: {{proxy_cpu_req|default('100m')}} + memory: {{proxy_mem_req|default('100Mi')}} + limits: + cpu: {{proxy_cpu_limit|default('1')}} + memory: {{proxy_mem_limit|default('1024Mi')}} + +repository: {{proxy_repository|default('proxy')}} +image_tag: {{ image_tag }} + +proxyconfig: |- + {% if proto=='https' %} + server { + listen 80; + listen [::]:80; + server_name {{ proxy_server_name }}; + {# + custom nginx server config section + eg: + nginx_server_config: | + if ($allowed_country = no) { + return 444; + } + #} +{% if nginx_server_config is defined and nginx_server_config %} + {{ nginx_server_config | indent( width=4, indentfirst=True) }} +{% endif %} + # Limitting open connection per ip + limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; + return 301 https://{{ proxy_server_name }}$request_uri; + } + {% endif %} + server { + {% if proto=='http' %} + listen 80; + listen [::]:80; + {% else %} + listen 443 ssl; + ssl_certificate /etc/secrets/site.crt; + ssl_certificate_key /etc/secrets/site.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA HIGH !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS"; + {% endif %} + server_name *.{{ proxy_server_name }} {{ proxy_server_name }}; + {# + custom nginx server config section + eg: + nginx_server_config: | + if ($allowed_country = no) { + return 444; + } + #} +{% if nginx_server_config is defined and nginx_server_config %} + {{ nginx_server_config | indent( width=6, indentfirst=True) }} +{% endif %} + # Limitting open connection per ip + limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-SSL on; + proxy_set_header X-Forwarded-Proto $scheme; + ignore_invalid_headers off; #pass through headers from Jenkins which are considered invalid by Nginx server. + resolver {{ kube_dns_ip }} valid=30s; + # Mobile Devices Refresh token Endpoints + location ~* ^/auth/v1/refresh/token { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + # Admin API Endpoints for sunbird realm fpr forgot password flow + location ~ /auth/admin/realms/sunbird/users/ { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Sunbird realm keycloak API endpoints + location ~ /auth/realms/sunbird/(get-required-action-link|login-actions/(action-token|authenticate|required-action)|protocol/openid-connect/(auth|certs|logout|token|userinfo)|.well-known/openid-configuration) { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Static Assets for keycloak endpoints with caching + location ~ /auth/(resources/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))|welcome-content/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))) { + # Enabling caching + proxy_cache_key $proxy_host$request_uri; + proxy_cache proxy_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # This is Caching mechanism for POST requests location search + location ~ /learner/data/v1/location/search { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + # Caching for content consumption + location ~ /api/(content/v1/read|course/v1/hierarchy|course/v1/batch/read) { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache content_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 14400s; + # Increasing the proxy buffer size + proxy_buffer_size 16k; + proxy_busy_buffers_size 16k; + rewrite ^/api/(.*) /$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://kong; + } + # This is Caching mechanism for Content search + location ~ /api/content/v1/search { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache content_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 14400s; + # Increasing the proxy buffer size + proxy_buffer_size 16k; + proxy_busy_buffers_size 16k; + rewrite ^/api/(.*) /$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://kong; + } + # This is Caching mechanism for POST requests + location ~ /api/org/v1/search|/api/data/v1/(form/read|location/search) { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_methods GET HEAD POST; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + rewrite ^/api/(.*) /$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + location ~ /api/(framework/v1/read|data/v1/system/settings/get) { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + rewrite ^/api/(.*) /$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + location /api/ { + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id, Content-Encoding"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + if ( $arg_eHVyhwSdt ) { + set $custom_header "Bearer $arg_eHVyhwSdt"; + } + if ( $http_authorization ) { + set $custom_header "$http_authorization"; + } + include /etc/nginx/defaults.d/compression.conf; + proxy_set_header Authorization $custom_header; + rewrite ^/api/(.*) /$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + # Oauth2 config + location /oauth2/ { + set $target http://oauth2-proxy.logging.svc.cluster.local; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Auth-Request-Redirect $request_uri; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass $target; + } + location = /oauth2/auth { + set $target http://oauth2-proxy.logging.svc.cluster.local; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + # nginx auth_request includes headers but not body + proxy_set_header Content-Length ""; + proxy_pass_request_body off; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass $target; + } + location /dashboard/ { + auth_request /oauth2/auth; + error_page 401 = /oauth2/sign_in; + # Setting target url + auth_request_set $target http://{{ kibana_service }}; + # pass information via X-User and X-Email headers to backend, + # requires running with --set-xauthrequest flag + auth_request_set $user $upstream_http_x_auth_request_user; + auth_request_set $email $upstream_http_x_auth_request_email; + proxy_set_header X-User $user; + proxy_set_header X-Email $email; + # if you enabled --cookie-refresh, this is needed for it to work with auth_request + auth_request_set $auth_cookie $upstream_http_set_cookie; + add_header Set-Cookie $auth_cookie; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass $target; + } + location /grafana/ { + set $target http://prometheus-operator-grafana.monitoring.svc.cluster.local; + rewrite ^/grafana/(.*) /$1 break; + proxy_pass $target; + } + location /encryption/ { + set $target http://encryption.{{ namespace }}.svc.cluster.local; + rewrite ^/encryption/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 1; + proxy_send_timeout 30; + proxy_read_timeout 40; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $http_x_forwarded_for; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass $target; + } + location /badging/ { + set $target http://badger-service.{{ namespace }}.svc.cluster.local:8004; + rewrite ^/badging/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 1; + proxy_send_timeout 30; + proxy_read_timeout 40; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass $target; + } + location /discussions/ { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + proxy_set_header X-NginX-Proxy true; + proxy_redirect off; + # Socket.IO Support + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + set $target http://nodebb-service.{{ namespace }}.svc.cluster.local:4567; + #rewrite ^/discussions/(.*) /$1 break; + proxy_pass $target; + } + location ~* ^/assets/public/(.*) { + # Enabling cache for Response code 200 + expires 1M; + add_header Pragma public; + add_header Cache-Control "public"; + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $bucket "{{upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*"; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$bucket/$url_full; + } + location ~* ^/content/preview/(.*) { + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $s3_bucket "{{plugin_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{plugin_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/v3/preview/$url_full; + } + location ~ /content-editor/telemetry|collection-editor/telemetry { + rewrite ^/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + location ~* ^/content-editor/(.*) { + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $s3_bucket "{{plugin_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{plugin_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/content-editor/$url_full; + } + + location ~* ^/discussion-ui/(.*) { + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + set $s3_bucket "{{discussion_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{discussion_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/discussion-ui/$url_full; + } + + location ~* ^/collection-editor/(.*) { + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $s3_bucket "{{plugin_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{plugin_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/collection-editor/$url_full; + } + location ~* ^/generic-editor/(.*) { + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $s3_bucket "{{plugin_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{plugin_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/generic-editor/$url_full; + } + location ~* ^/content-plugins/(.*) { + # Enabling cache for Response code 200 + expires 1M; + add_header Pragma public; + add_header Cache-Control "public"; + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $s3_bucket "{{plugin_upstream_url}}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{plugin_upstream_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*"; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$s3_bucket/content-plugins/$url_full; + } + location /thirdparty { + # Enabling cache for Response code 200 + expires 1M; + add_header Pragma public; + add_header Cache-Control "public"; + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + rewrite ^/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + location ~* ^/desktop/(.*) { + # Enabling cache for Response code 200 + expires 1M; + add_header Pragma public; + add_header Cache-Control "public"; + # Enabling compression + gzip on; + gzip_min_length 100000; + gzip_proxied expired no-cache no-store private auth; + gzip_types application/javascript application/x-javascript text/css text/javascript; + if ($request_method = OPTIONS ) { + add_header Access-Control-Allow-Origin "*" ; + add_header Access-Control-Allow-Methods "GET, OPTIONS, PATCH, POST"; + add_header Access-Control-Allow-Headers "Access-Control-Allow-Origin, Authorization, Content-Type, user-id"; + # add_header Access-Control-Allow-Credentials "true"; + add_header Content-Length 0; + add_header Content-Type text/plain; + return 200; + } + set $offline_bucket "{{ sunbird_offline_azure_storage_account_url }}"; + set $url_full '$1'; + proxy_http_version 1.1; + proxy_set_header Host "{{sunbird_offline_azure_storage_account_url.split('/')[0]|lower}}"; + proxy_set_header Authorization ''; + proxy_hide_header Access-Control-Allow-Origin; + proxy_hide_header Access-Control-Allow-Methods; + proxy_hide_header x-amz-id-2; + proxy_hide_header x-amz-request-id; + proxy_hide_header Set-Cookie; + proxy_ignore_headers "Set-Cookie"; + proxy_buffering off; + proxy_intercept_errors on; + add_header Access-Control-Allow-Origin "*"; + add_header Access-Control-Allow-Methods GET; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass https://$offline_bucket/$url_full; + } + # compression for svg certs download + location /api/certreg/v2/certs/download { + rewrite ^/api/(.*) /$1 break; + include /etc/nginx/defaults.d/compression.conf; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + location /learner/certreg/v2/certs/download { + # Compression + gzip on; + gzip_comp_level 5; + gzip_min_length 50000; # 50KB + gzip_proxied any; + gzip_vary on; + # Content types for compression + gzip_types + application/atom+xml + application/javascript + application/json + application/ld+json + application/manifest+json + application/rss+xml + application/vnd.geo+json + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/bmp + image/svg+xml + image/x-icon + text/cache-manifest + text/css + text/plain + add_header test hello; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + location / { + rewrite ^/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + location /v3/device/register { + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + proxy_set_header Connection ""; + rewrite ^/v3/device/register/(.*) /v3/device/register/$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + } + location /action/data/v3/metrics { + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + proxy_set_header Connection ""; + rewrite ^/action/data/v3/metrics/(.*) /data/v3/metrics/$1 break; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + } + location ~ /resourcebundles/v1/read|/learner/data/v1/(role/read|system/settings/get)|/v1/tenant/info { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://player; + } + location /api/channel/v1/read { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache framework_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 43200; + rewrite ^/api/channel/v1/read/(.*) /channel/v1/read/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + location ~ ^/chatapi/ { + set $target http://router-service.{{ namespace }}.svc.cluster.local:8000; + rewrite ^/chatapi/(.*) /$1 break; + proxy_pass $target; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + } + location /oauth2callback { + return 200 'OK'; + add_header Content-Type text/plain; + } + {# Including custom configuration #} + {{ proxy_custom_config }}} + +nginxconfig: | + user nginx; + worker_processes {{nginx_worker_processes | d("auto")}}; + {# + Can add custom modules like + eg: + nginx_modules: | + load_module modules/ngx_http_geoip2_module.so; + load_module modules/ngx_stream_geoip2_module.so; + #} +{% if nginx_modules is defined and nginx_modules %} + {{ nginx_modules | indent( width=2, indentfirst=True) }} +{% endif %} + error_log /var/log/nginx/error.log warn; + pid /var/run/nginx.pid; + events { + worker_connections 10000; + } + http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + resolver {{ kube_dns_ip }} valid=30s; + {# + This is to define custom nginx_http_configs + for example + nginx_http_config: | + geoip2 /usr/local/share/GeoLite2-Country.mmdb { + $geoip2_data_country_iso_code country iso_code; + } + map $geoip2_data_country_iso_code $allowed_country { + default no; + IN no; + } + #} +{% if nginx_http_config is defined and nginx_http_config %} + {{ nginx_http_config | indent( width=7, indentfirst=True) }} +{% endif %} + lua_load_resty_core off; + log_format main '{{ nginx_client_public_ip_header | d('$remote_addr') }} - $remote_user [$time_local] ' + '"$request" $status $request_length $body_bytes_sent' + ' $request_time $upstream_response_time $pipe' + ' "$http_referer" "$http_user_agent" "$sb_request_id"' + ' "$http_x_device_id" "$http_x_channel_id" "$http_x_app_id"' + ' "$http_x_app_ver" "$http_x_session_id" {{nginx_additional_log_fields | default("")}}'; + access_log /var/log/nginx/access.log main; + # Shared dictionary to store metrics + lua_shared_dict prometheus_metrics 100M; + lua_package_path "/etc/nginx/lua_modules/?.lua"; + # Defining request_id + # If the client send request_id it should be preffered over the default one + map $http_x_request_id $sb_request_id { + default $http_x_request_id; + '' $request_id; + } + # Defining upstream cache status for nginx metrics + map $upstream_cache_status $cache_status { + default $upstream_cache_status; + '' "NONE"; + } + # Defining metrics + init_worker_by_lua_block { + prometheus = require("prometheus").init("prometheus_metrics") + metric_requests = prometheus:counter( + "nginx_http_requests_total", "Number of HTTP requests", {"host", "status", "request_method", "cache_status"}) + metric_latency = prometheus:histogram( + "nginx_http_request_duration_seconds", "HTTP request latency", {"host"}) + metric_connections = prometheus:gauge( + "nginx_http_connections", "Number of HTTP connections", {"state"}) + } + log_by_lua_block { + metric_requests:inc(1, {ngx.var.server_name, ngx.var.status, ngx.var.request_method, ngx.var.cache_status }) + metric_latency:observe(tonumber(ngx.var.request_time), {ngx.var.server_name}) + } + header_filter_by_lua_block { + ngx.header["server"] = nil + } + sendfile on; + #tcp_nopush on; + client_max_body_size 60M; + keepalive_timeout 65s; + keepalive_requests 200; + # Nginx connection limit per ip + limit_conn_zone $binary_remote_addr zone=limitbyaddr:10m; + limit_conn_status 429; + upstream kong { + server kong:8000; + keepalive 1000; + } + upstream encryption { + server enc-service:8013; + keepalive 1000; + } + upstream keycloak { + server {{ keycloak_url.split('//')[-1] }}; + keepalive 1000; + } + upstream player { + server player:3000; + keepalive 1000; + } + include /etc/nginx/defaults.d/*.conf; + include /etc/nginx/conf.d/*.conf; + ################# + # Caching Block # + ################# + # + # Keywords + # + # proxy_cache_path: path to store the cache content + # level: how many directories we need, 1:2 means 1 parent directory, and another child directory before the cache content. + # keys_zone: name of the cache and size of the keys store in RAM; 1‑MB zone can store data for about 8,000 keys + # max_size: size of the cache content in disk + # inactive: specifies how long an item can remain in the cache without being accessed. This doesn't value expiry time of cache. So keep it more than the expiry. + # use_temp_path: do we have to write the cache to a temp path first? This will reduce the performance. + # + # caching for images and files + proxy_cache_path /tmp/proxy_cache levels=1:2 keys_zone=tmp_cache:5m max_size=10m inactive=300m use_temp_path=off; + # caching for apis + proxy_cache_path /tmp/api_cache levels=1:2 keys_zone=proxy_cache:5m max_size=600m inactive=1400m use_temp_path=off; + # cache framework + proxy_cache_path /tmp/framework_cache levels=1:2 keys_zone=framework_cache:5m max_size=700m inactive=300m use_temp_path=off; + # cache content + proxy_cache_path /tmp/content_cache levels=1:2 keys_zone=content_cache:100m max_size=3000m inactive=600m use_temp_path=off; + # cache content metadata + proxy_cache_path /tmp/content_metadata levels=1:2 keys_zone=content_metadata:100m max_size=1000m inactive=300m use_temp_path=off; + + server { + listen 9145; + location /metrics { + content_by_lua_block { + metric_connections:set(ngx.var.connections_reading, {"reading"}) + metric_connections:set(ngx.var.connections_waiting, {"waiting"}) + metric_connections:set(ngx.var.connections_writing, {"writing"}) + prometheus:collect() + } + } + } + } + +keycloakconf: | + server { + listen 80; + listen [::]:80; + server_name {{ merge_proxy_server_name }}; + # Limitting open connection per ip + limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; + return 301 https://{{ merge_proxy_server_name }}$request_uri; + } + server { + listen 443 ssl; + ssl_certificate /etc/secrets-merge/tls.crt; + ssl_certificate_key /etc/secrets-merge/tls.key; + server_name {{ merge_proxy_server_name }}; + # Limitting open connection per ip + limit_conn limitbyaddr {{ nginx_per_ip_connection_limit }}; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-SSL on; + proxy_set_header X-Forwarded-Proto $scheme; + ignore_invalid_headers off; #pass through headers from Jenkins which are considered invalid by Nginx server. + resolver 127.0.0.11 valid=5s; + # Refresh token endpoint being routed to kong + location ~* ^/auth/v1/refresh/token { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $sb_request_id; + proxy_pass http://kong; + } + # Admin API Endpoints for sunbird realm fpr forgot password flow + location ~ /auth/admin/realms/sunbird/users/ { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Sunbird realm keycloak API endpoints + location ~ /auth/realms/sunbird/(get-required-action-link|login-actions/(action-token|authenticate|required-action)|protocol/openid-connect/(auth|certs|logout|token|userinfo)|.well-known/openid-configuration) { + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header X-Request-ID $sb_request_id; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + # Static Assets for keycloak endpoints with caching + location ~ /auth/(resources/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))|welcome-content/(.+\.(png|svg|ico|js|eot|ttf|woff|woff2|css))) { + # Enabling compression + include /etc/nginx/defaults.d/compression.conf; + # Enabling caching + # caching include Accept-Encoding header also, to provide gziped or plain content as per request + proxy_cache_key "$http_accept_encoding|$request_uri|$request_body"; + proxy_cache proxy_cache; + add_header X-Proxy-Cache $upstream_cache_status; + add_header X-Proxy-Cache-Date $upstream_http_date; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + proxy_cache_revalidate on; + proxy_cache_background_update on; + proxy_cache_lock on; + proxy_cache_valid 200 14400; + rewrite ^/auth/(.*) /auth/$1 break; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-For {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_http_version 1.1; + proxy_pass http://keycloak; + } + location / { + rewrite ^/(.*) /$1 break; + proxy_set_header Host $host; + proxy_set_header X-Real-IP {{ nginx_client_public_ip_header | d('$remote_addr') }}; + proxy_set_header X-Scheme $scheme; + proxy_connect_timeout 5; + proxy_send_timeout 60; + proxy_read_timeout 70; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_http_version 1.1; + proxy_set_header X-Request-ID $request_id; + proxy_pass http://player; + } + } + +compressionConfig: |- + # Compression + gzip on; + gzip_comp_level 5; + gzip_min_length 256; # 256Bytes + gzip_proxied any; + gzip_vary on; + # Content types for compression + gzip_types + application/atom+xml + application/javascript + application/json + application/ld+json + application/manifest+json + application/rss+xml + application/vnd.geo+json + application/vnd.ms-fontobject + application/x-font-ttf + application/x-web-app-manifest+json + application/xhtml+xml + application/xml + font/opentype + image/bmp + image/svg+xml + image/x-icon + text/cache-manifest + text/css + text/plain + ; + +serviceMonitor: + enabled: true + labels: # labels with which the prometheus choose the serviceMonitor + app: prometheus-operator + release: prometheus-operator diff --git a/kubernetes/helm_charts/core/nodebb/.helmignore b/kubernetes/helm_charts/core/nodebb/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/nodebb/Chart.yaml b/kubernetes/helm_charts/core/nodebb/Chart.yaml new file mode 100644 index 000000000..92b3d1bb7 --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: nodebb +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/nodebb/templates/_helpers.tpl b/kubernetes/helm_charts/core/nodebb/templates/_helpers.tpl new file mode 100644 index 000000000..83b59b74a --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "print.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "print.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "print.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "print.labels" -}} +app.kubernetes.io/name: {{ include "print.name" . }} +helm.sh/chart: {{ include "print.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/nodebb/templates/configmap.yaml b/kubernetes/helm_charts/core/nodebb/templates/configmap.yaml new file mode 100644 index 000000000..bdec29c8d --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/templates/configmap.yaml @@ -0,0 +1 @@ +# This file will get replaced at runtime diff --git a/kubernetes/helm_charts/core/nodebb/templates/deployment.yaml b/kubernetes/helm_charts/core/nodebb/templates/deployment.yaml new file mode 100644 index 000000000..46b2eb8a4 --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/templates/deployment.yaml @@ -0,0 +1,86 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: nodebb-config-json + configMap: + name: nodebb-config-json + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + volumeMounts: + - name: nodebb-config-json + mountPath: /usr/src/app/config.json + subPath: config.json + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nodebb-config-json +data: + config.json: |- + { + "url": "{{ .Values.nodebb_url }}", + "secret": "100101", + "database": "mongo", + "port": "4567", + "mongo": { + "uri": "mongodb://{{ .Values.mongo_nodebb_host }}", + "username": "", + "password": "", + } + } + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.port }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/nodebb/values.j2 b/kubernetes/helm_charts/core/nodebb/values.j2 new file mode 100644 index 000000000..1d7089ab1 --- /dev/null +++ b/kubernetes/helm_charts/core/nodebb/values.j2 @@ -0,0 +1,27 @@ +#jinja2:lstrip_blocks: True + +### Default variable file for nodebb-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{nodebb_replicacount|default(1)}} +repository: {{nodebb_repository|default('nodebb')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{nodebb_cpu_req|default('50m')}} + memory: {{nodebb_mem_req|default('50Mi')}} + limits: + cpu: {{nodebb_cpu_limit|default('1')}} + memory: {{nodebb_mem_limit|default('1024Mi')}} +network: + port: 4567 +strategy: + type: RollingUpdate + maxsurge: {{ nodebb_maxsurge|default('25%') }} + maxunavailable: {{ nodebb_maxunavailable|default('25%') }} + +nodebb_url: {{ sunbird_nodebb_url }} +mongo_nodebb_host: {{ mongo_nodebb_host }} diff --git a/kubernetes/helm_charts/core/notification/Chart.yaml b/kubernetes/helm_charts/core/notification/Chart.yaml new file mode 100644 index 000000000..e4f958f07 --- /dev/null +++ b/kubernetes/helm_charts/core/notification/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: notification +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/notification/templates/configmap.yaml b/kubernetes/helm_charts/core/notification/templates/configmap.yaml new file mode 100644 index 000000000..2ebdaf39b --- /dev/null +++ b/kubernetes/helm_charts/core/notification/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.notificationenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/notification/templates/deployment.yaml b/kubernetes/helm_charts/core/notification/templates/deployment.yaml new file mode 100644 index 000000000..a82590540 --- /dev/null +++ b/kubernetes/helm_charts/core/notification/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/notification/values.j2 b/kubernetes/helm_charts/core/notification/values.j2 new file mode 100644 index 000000000..a37b1c8d7 --- /dev/null +++ b/kubernetes/helm_charts/core/notification/values.j2 @@ -0,0 +1,65 @@ +### Default variable file for enc-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{notification_replicacount|default(1)}} +repository: {{notification_repository|default('notification_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{notification_cpu_req|default('50m')}} + memory: {{notification_mem_req|default('50Mi')}} + limits: + cpu: {{notification_cpu_limit|default('1')}} + memory: {{notification_mem_limit|default('500Mi')}} +network: + port: 9012 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ notification_maxsurge|default('25%') }} + maxunavailable: {{ notification_maxunavailable|default('25%') }} + +notification: + readinessprobe: + enabled: true + livenessprobe: + enabled: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 10 + +notification: + readinessprobe: + enabled: true + livenessprobe: + enabled: true + readinessProbe: + httpGet: + path: /health + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 5 + successThreshold: 1 + livenessProbe: + httpGet: + path: /service/health + port: 9000 + initialDelaySeconds: 60 + periodSeconds: 10 diff --git a/kubernetes/helm_charts/core/player/.helmignore b/kubernetes/helm_charts/core/player/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/player/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/player/Chart.yaml b/kubernetes/helm_charts/core/player/Chart.yaml new file mode 100644 index 000000000..673a6a73f --- /dev/null +++ b/kubernetes/helm_charts/core/player/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: player +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/player/templates/_helpers.tpl b/kubernetes/helm_charts/core/player/templates/_helpers.tpl new file mode 100644 index 000000000..2317f3842 --- /dev/null +++ b/kubernetes/helm_charts/core/player/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "player.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "player.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "player.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "player.labels" -}} +app.kubernetes.io/name: {{ include "player.name" . }} +helm.sh/chart: {{ include "player.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/player/templates/configmap.yaml b/kubernetes/helm_charts/core/player/templates/configmap.yaml new file mode 100644 index 000000000..37553c79c --- /dev/null +++ b/kubernetes/helm_charts/core/player/templates/configmap.yaml @@ -0,0 +1,11 @@ +--- +{{- $keys := .Files.Glob "keys/*" }} +{{ if $keys }} +apiVersion: v1 +kind: Secret +metadata: + name: player-access-keys +type: Opaque +data: +{{ (.Files.Glob "keys/*").AsSecrets | indent 2 }} +{{ end }} diff --git a/kubernetes/helm_charts/core/player/templates/deployment.yaml b/kubernetes/helm_charts/core/player/templates/deployment.yaml new file mode 100644 index 000000000..642ecf275 --- /dev/null +++ b/kubernetes/helm_charts/core/player/templates/deployment.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} +{{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if or .Values.sunbird_portal_player_cdn_enabled $keys }} + volumeMounts: +{{- end }} +{{- if .Values.sunbird_portal_player_cdn_enabled }} + - name: player-cdn-config + mountPath: /home/sunbird/app_dist/dist/index_cdn.ejs + subPath: index_cdn.ejs +{{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - mountPath: {{ .Values.player_access_basepath }} + name: access-keys +{{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if or .Values.sunbird_portal_player_cdn_enabled $keys }} + volumes: +{{- end }} +{{- if .Values.sunbird_portal_player_cdn_enabled }} + - name: player-cdn-config + configMap: + name: player-cdn-config +{{- end }} +{{- $keys := .Files.Glob "keys/*" }} +{{- if $keys }} + - name: access-keys + secret: + secretName: player-access-keys +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/player/values.j2 b/kubernetes/helm_charts/core/player/values.j2 new file mode 100644 index 000000000..c21f6302b --- /dev/null +++ b/kubernetes/helm_charts/core/player/values.j2 @@ -0,0 +1,29 @@ +### Default variable file for player-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{player_replicacount|default(1)}} +repository: {{player_repository|default('player')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{player_cpu_req|default('100m')}} + memory: {{player_mem_req|default('100Mi')}} + limits: + cpu: {{player_cpu_limit|default('1')}} + memory: {{player_mem_limit|default('1024Mi')}} +network: + port: 3000 + targetport: 3000 +strategy: + type: RollingUpdate + maxsurge: {{ player_maxsurge|default('25%') }} + maxunavailable: {{ player_maxunavailable|default('25%') }} + +sunbird_portal_player_cdn_enabled: {{sunbird_portal_player_cdn_enabled|lower}} + +{{ player_liveness_readiness | to_nice_yaml }} + +player_access_basepath: {{ player_access_basepath | default('/home/sunbird/app_dist/keys/') }} diff --git a/kubernetes/helm_charts/core/print/.helmignore b/kubernetes/helm_charts/core/print/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/print/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/print/Chart.yaml b/kubernetes/helm_charts/core/print/Chart.yaml new file mode 100644 index 000000000..8d0bbc55a --- /dev/null +++ b/kubernetes/helm_charts/core/print/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: print +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/print/templates/_helpers.tpl b/kubernetes/helm_charts/core/print/templates/_helpers.tpl new file mode 100644 index 000000000..83b59b74a --- /dev/null +++ b/kubernetes/helm_charts/core/print/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "print.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "print.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "print.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "print.labels" -}} +app.kubernetes.io/name: {{ include "print.name" . }} +helm.sh/chart: {{ include "print.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/print/templates/configmap.yaml b/kubernetes/helm_charts/core/print/templates/configmap.yaml new file mode 100644 index 000000000..bdec29c8d --- /dev/null +++ b/kubernetes/helm_charts/core/print/templates/configmap.yaml @@ -0,0 +1 @@ +# This file will get replaced at runtime diff --git a/kubernetes/helm_charts/core/print/templates/deployment.yaml b/kubernetes/helm_charts/core/print/templates/deployment.yaml new file mode 100644 index 000000000..11be85131 --- /dev/null +++ b/kubernetes/helm_charts/core/print/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.port }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/print/values.j2 b/kubernetes/helm_charts/core/print/values.j2 new file mode 100644 index 000000000..b8f9e06f3 --- /dev/null +++ b/kubernetes/helm_charts/core/print/values.j2 @@ -0,0 +1,26 @@ +#jinja2:lstrip_blocks: True + +### Default variable file for print-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{print_replicacount|default(1)}} +repository: {{print_repository|default('print-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{print_cpu_req|default('50m')}} + memory: {{print_mem_req|default('50Mi')}} + limits: + cpu: {{print_cpu_limit|default('1')}} + memory: {{print_mem_limit|default('1024Mi')}} +network: + port: 5000 +strategy: + type: RollingUpdate + maxsurge: {{ print_maxsurge|default('25%') }} + maxunavailable: {{ print_maxunavailable|default('25%') }} + +{{ print_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/reloader/.helmignore b/kubernetes/helm_charts/core/reloader/.helmignore new file mode 100755 index 000000000..9e1690881 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/.helmignore @@ -0,0 +1,2 @@ +# OWNERS file for Kubernetes +OWNERS diff --git a/kubernetes/helm_charts/core/reloader/Chart.yaml b/kubernetes/helm_charts/core/reloader/Chart.yaml new file mode 100755 index 000000000..b4995a9a8 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +appVersion: v0.0.49 +description: Reloader chart that runs on kubernetes +home: https://github.com/stakater/Reloader +icon: https://raw.githubusercontent.com/stakater/Reloader/master/assets/web/reloader-round-100px.png +keywords: +- Reloader +- kubernetes +maintainers: +- email: hello@stakater.com + name: Stakater +- email: rasheed@aurorasolutions.io + name: rasheedamir +- email: waseemhassan@stakater.com + name: waseem-h +- email: faizan.ahmad55@outlook.com + name: faizanahmad055 +- email: ali.kahoot@aurorasolutions.io + name: kahootali +- email: ahmad@aurorasolutions.io + name: ahmadiq +- email: ahsanmuhammad1@outlook.com + name: ahsan-storm +name: reloader +sources: +- https://github.com/stakater/IngressMonitorController +version: v0.0.49 diff --git a/kubernetes/helm_charts/core/reloader/templates/NOTES.txt b/kubernetes/helm_charts/core/reloader/templates/NOTES.txt new file mode 100755 index 000000000..f2a38752a --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/NOTES.txt @@ -0,0 +1,7 @@ +- For a `Deployment` called `foo` have a `ConfigMap` called `foo-configmap`. Then add this annotation to main metadata of your `Deployment` + configmap.reloader.stakater.com/reload: "foo-configmap" + +- For a `Deployment` called `foo` have a `Secret` called `foo-secret`. Then add this annotation to main metadata of your `Deployment` + secret.reloader.stakater.com/reload: "foo-secret" + +- After successful installation, your pods will get rolling updates when a change in data of configmap or secret will happen. diff --git a/kubernetes/helm_charts/core/reloader/templates/_helpers.tpl b/kubernetes/helm_charts/core/reloader/templates/_helpers.tpl new file mode 100755 index 000000000..15ca27ef7 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/_helpers.tpl @@ -0,0 +1,35 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} + +{{- define "reloader-name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" | lower -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "reloader-fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "reloader-labels.chart" -}} +app: {{ template "reloader-fullname" . }} +chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" +release: {{ .Release.Name | quote }} +heritage: {{ .Release.Service | quote }} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "reloader-serviceAccountName" -}} +{{- if .Values.reloader.serviceAccount.create -}} + {{ default (include "reloader-fullname" .) .Values.reloader.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.reloader.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/core/reloader/templates/clusterrole.yaml b/kubernetes/helm_charts/core/reloader/templates/clusterrole.yaml new file mode 100755 index 000000000..8d51ef406 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/clusterrole.yaml @@ -0,0 +1,62 @@ +{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: +{{- if .Values.reloader.ignoreSecrets }}{{- else }} + - secrets +{{- end }} +{{- if .Values.reloader.ignoreConfigMaps }}{{- else }} + - configmaps +{{- end }} + verbs: + - list + - get + - watch +{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.isOpenshift) }} + - apiGroups: + - "apps.openshift.io" + - "" + resources: + - deploymentconfigs + verbs: + - list + - get + - update + - patch +{{- end }} + - apiGroups: + - "apps" + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - list + - get + - update + - patch + - apiGroups: + - "extensions" + resources: + - deployments + - daemonsets + verbs: + - list + - get + - update + - patch +{{- end }} diff --git a/kubernetes/helm_charts/core/reloader/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/core/reloader/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..28c9d4b91 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "reloader-fullname" . }}-role +subjects: + - kind: ServiceAccount + name: {{ template "reloader-serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/core/reloader/templates/deployment.yaml b/kubernetes/helm_charts/core/reloader/templates/deployment.yaml new file mode 100755 index 000000000..c03a86f50 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/deployment.yaml @@ -0,0 +1,129 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: +{{- if .Values.reloader.deployment.annotations }} + annotations: +{{ toYaml .Values.reloader.deployment.annotations | indent 4 }} +{{- end }} + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.deployment.labels }} +{{ toYaml .Values.reloader.deployment.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }} +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: {{ template "reloader-fullname" . }} + release: {{ .Release.Name | quote }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 6 }} +{{- end }} + template: + metadata: +{{- if .Values.reloader.deployment.pod.annotations }} + annotations: +{{ toYaml .Values.reloader.deployment.pod.annotations | indent 8 }} +{{- end }} + labels: +{{ include "reloader-labels.chart" . | indent 8 }} +{{- if .Values.reloader.deployment.labels }} +{{ toYaml .Values.reloader.deployment.labels | indent 8 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 8 }} +{{- end }} + spec: + {{- if .Values.reloader.deployment.nodeSelector }} + nodeSelector: +{{ toYaml .Values.reloader.deployment.nodeSelector | indent 8 }} + {{- end }} + {{- if .Values.reloader.deployment.affinity }} + affinity: +{{ toYaml .Values.reloader.deployment.affinity | indent 8 }} + {{- end }} + {{- if .Values.reloader.deployment.tolerations }} + tolerations: +{{ toYaml .Values.reloader.deployment.tolerations | indent 8 }} + {{- end }} + containers: + - env: + {{- range $name, $value := .Values.reloader.deployment.env.open }} + {{- if not (empty $value) }} + - name: {{ $name | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- $secret_name := include "reloader-fullname" . }} + {{- range $name, $value := .Values.reloader.deployment.env.secret }} + {{- if not ( empty $value) }} + - name: {{ $name | quote }} + valueFrom: + secretKeyRef: + name: {{ $secret_name }} + key: {{ $name | quote }} + {{- end }} + {{- end }} + {{- range $name, $value := .Values.reloader.deployment.env.field }} + {{- if not ( empty $value) }} + - name: {{ $name | quote }} + valueFrom: + fieldRef: + fieldPath: {{ $value | quote}} + {{- end }} + {{- end }} + {{- if eq .Values.reloader.watchGlobally false }} + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- end }} + image: "{{ .Values.reloader.deployment.image.name }}:{{ .Values.reloader.deployment.image.tag }}" + imagePullPolicy: {{ .Values.reloader.deployment.image.pullPolicy }} + name: {{ template "reloader-fullname" . }} + {{- if eq .Values.reloader.readOnlyRootFileSystem true }} + volumeMounts: + - mountPath: /tmp/ + name: tmp-volume + {{- end }} + args: + {{- if .Values.reloader.ignoreSecrets }} + - "--resources-to-ignore=secrets" + {{- end }} + {{- if eq .Values.reloader.ignoreConfigMaps true }} + - "--resources-to-ignore=configMaps" + {{- end }} + + {{- if .Values.reloader.custom_annotations }} + {{- if .Values.reloader.custom_annotations.configmap }} + - "--configmap-annotation" + - "{{ .Values.reloader.custom_annotations.configmap }}" + {{- end }} + {{- if .Values.reloader.custom_annotations.secret }} + - "--secret-annotation" + - "{{ .Values.reloader.custom_annotations.secret }}" + {{- end }} + {{- if .Values.reloader.custom_annotations.auto }} + - "--auto-annotation" + - "{{ .Values.reloader.custom_annotations.auto }}" + {{- end }} + {{- end }} + + {{- if .Values.reloader.deployment.resources }} + resources: +{{ toYaml .Values.reloader.deployment.resources | indent 10 }} + {{- end }} +{{- if .Values.reloader.deployment.securityContext }} + securityContext: {{ toYaml .Values.reloader.deployment.securityContext | nindent 8 }} +{{- end }} + serviceAccountName: {{ template "reloader-serviceAccountName" . }} + {{- if eq .Values.reloader.readOnlyRootFileSystem true }} + volumes: + - emptyDir: {} + name: tmp-volume + {{- end }} diff --git a/kubernetes/helm_charts/core/reloader/templates/role.yaml b/kubernetes/helm_charts/core/reloader/templates/role.yaml new file mode 100755 index 000000000..5827f5cdc --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/role.yaml @@ -0,0 +1,62 @@ +{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + resources: +{{- if .Values.reloader.ignoreSecrets }}{{- else }} + - secrets +{{- end }} +{{- if .Values.reloader.ignoreConfigMaps }}{{- else }} + - configmaps +{{- end }} + verbs: + - list + - get + - watch +{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.reloader.isOpenshift) }} + - apiGroups: + - "apps.openshift.io" + - "" + resources: + - deploymentconfigs + verbs: + - list + - get + - update + - patch +{{- end }} + - apiGroups: + - "apps" + resources: + - deployments + - daemonsets + - statefulsets + verbs: + - list + - get + - update + - patch + - apiGroups: + - "extensions" + resources: + - deployments + - daemonsets + verbs: + - list + - get + - update + - patch +{{- end }} diff --git a/kubernetes/helm_charts/core/reloader/templates/rolebinding.yaml b/kubernetes/helm_charts/core/reloader/templates/rolebinding.yaml new file mode 100755 index 000000000..94fb1f838 --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/rolebinding.yaml @@ -0,0 +1,23 @@ +{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.rbac.labels }} +{{ toYaml .Values.reloader.rbac.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-fullname" . }}-role-binding + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "reloader-fullname" . }}-role +subjects: + - kind: ServiceAccount + name: {{ template "reloader-serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/core/reloader/templates/serviceaccount.yaml b/kubernetes/helm_charts/core/reloader/templates/serviceaccount.yaml new file mode 100755 index 000000000..62e5f1a0a --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.reloader.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +metadata: + labels: +{{ include "reloader-labels.chart" . | indent 4 }} +{{- if .Values.reloader.serviceAccount.labels }} +{{ toYaml .Values.reloader.serviceAccount.labels | indent 4 }} +{{- end }} +{{- if .Values.reloader.matchLabels }} +{{ toYaml .Values.reloader.matchLabels | indent 4 }} +{{- end }} + name: {{ template "reloader-serviceAccountName" . }} +{{- end }} diff --git a/kubernetes/helm_charts/core/reloader/values.yaml b/kubernetes/helm_charts/core/reloader/values.yaml new file mode 100755 index 000000000..0973025cb --- /dev/null +++ b/kubernetes/helm_charts/core/reloader/values.yaml @@ -0,0 +1,88 @@ +# Generated from deployments/kubernetes/templates/chart/values.yaml.tmpl +global: + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + +kubernetes: + host: https://kubernetes.default + +reloader: + isOpenshift: false + ignoreSecrets: false + ignoreConfigMaps: false + watchGlobally: true + # Set to true if you have a pod security policy that enforces readOnlyRootFilesystem + readOnlyRootFileSystem: false + matchLabels: {} + deployment: + nodeSelector: + # cloud.google.com/gke-nodepool: default-pool + + # An affinity stanza to be applied to the Deployment. + # Example: + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "node-role.kubernetes.io/infra-worker" + # operator: "Exists" + affinity: {} + + # A list of tolerations to be applied to the Deployment. + # Example: + # tolerations: + # - key: "node-role.kubernetes.io/infra-worker" + # operator: "Exists" + # effect: "NoSchedule" + tolerations: [] + + labels: + provider: stakater + group: com.stakater.platform + version: v0.0.49 + image: + name: stakater/reloader + tag: "v0.0.49" + pullPolicy: IfNotPresent + # Support for extra environment variables. + env: + # Open supports Key value pair as environment variables. + open: + # secret supports Key value pair as environment variables. It gets the values based on keys from default reloader secret if any. + secret: + # field supports Key value pair as environment variables. It gets the values from other fields of pod. + field: + + # Specify resource requests/limits for the deployment. + # Example: + # resources: + # limits: + # cpu: "100m" + # memory: "512Mi" + # requests: + # cpu: "10m" + # memory: "128Mi" + resources: {} + pod: + annotations: {} + + rbac: + enabled: true + labels: {} + # Service account config for the agent pods + serviceAccount: + # Specifies whether a ServiceAccount should be created + create: true + labels: {} + # The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template + name: + # Optional flags to pass to the Reloader entrypoint + # Example: + # custom_annotations: + # configmap: "my.company.com/configmap" + # secret: "my.company.com/secret" + custom_annotations: {} diff --git a/kubernetes/helm_charts/core/report/Chart.yaml b/kubernetes/helm_charts/core/report/Chart.yaml new file mode 100644 index 000000000..3ef91a05f --- /dev/null +++ b/kubernetes/helm_charts/core/report/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: report +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/report/templates/_helpers.tpl b/kubernetes/helm_charts/core/report/templates/_helpers.tpl new file mode 100644 index 000000000..590205843 --- /dev/null +++ b/kubernetes/helm_charts/core/report/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "telemetry.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "telemetry.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "telemetry.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "telemetry.labels" -}} +app.kubernetes.io/name: {{ include "telemetry.name" . }} +helm.sh/chart: {{ include "telemetry.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/report/templates/configmap.yaml b/kubernetes/helm_charts/core/report/templates/configmap.yaml new file mode 100644 index 000000000..00fdbaa0c --- /dev/null +++ b/kubernetes/helm_charts/core/report/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.telemetryenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/report/templates/deployment.yaml b/kubernetes/helm_charts/core/report/templates/deployment.yaml new file mode 100644 index 000000000..a82590540 --- /dev/null +++ b/kubernetes/helm_charts/core/report/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/report/values.j2 b/kubernetes/helm_charts/core/report/values.j2 new file mode 100644 index 000000000..d4c73485c --- /dev/null +++ b/kubernetes/helm_charts/core/report/values.j2 @@ -0,0 +1,26 @@ + +### variables for report service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{report_replicacount|default(1)}} +repository: {{report_repository|default('report-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{report_cpu_req|default('100m')}} + memory: {{report_mem_req|default('100Mi')}} + limits: + cpu: {{report_cpu_limit|default('1')}} + memory: {{report_mem_limit|default('1024Mi')}} +network: + port: 3030 + targetport: 3030 +strategy: + type: RollingUpdate + maxsurge: {{ report_maxsurge|default('25%') }} + maxunavailable: {{ report_maxunavailable|default('25%') }} + +{{ report_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/search/Chart.yaml b/kubernetes/helm_charts/core/search/Chart.yaml new file mode 100644 index 000000000..a70e56598 --- /dev/null +++ b/kubernetes/helm_charts/core/search/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: search +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/search/templates/deployment.yaml b/kubernetes/helm_charts/core/search/templates/deployment.yaml new file mode 100644 index 000000000..34293c1af --- /dev/null +++ b/kubernetes/helm_charts/core/search/templates/deployment.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /home/sunbird/search-service-1.0-SNAPSHOT/config/application.conf + subPath: search-service_application.conf + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/search-service-1.0-SNAPSHOT/config/logback.xml + subPath: search-service_logback.xml + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/search/values.j2 b/kubernetes/helm_charts/core/search/values.j2 new file mode 100644 index 000000000..e1236cdfd --- /dev/null +++ b/kubernetes/helm_charts/core/search/values.j2 @@ -0,0 +1,28 @@ +### Default variable file for search-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{search_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{search_replicacount|default(1)}} +repository: {{search_repository|default('search-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{search_cpu_req|default('100m')}} + memory: {{search_mem_req|default('100Mi')}} + limits: + cpu: {{search_cpu_limit|default('1')}} + memory: {{search_mem_limit|default('1024Mi')}} +network: + port: 9004 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ search_maxsurge|default('25%') }} + maxunavailable: {{ search_maxunavailable|default('25%') }} + +{{ search_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/taxonomy/Chart.yaml b/kubernetes/helm_charts/core/taxonomy/Chart.yaml new file mode 100644 index 000000000..8462df87c --- /dev/null +++ b/kubernetes/helm_charts/core/taxonomy/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: taxonomy +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/taxonomy/templates/deployment.yaml b/kubernetes/helm_charts/core/taxonomy/templates/deployment.yaml new file mode 100644 index 000000000..6c45087c0 --- /dev/null +++ b/kubernetes/helm_charts/core/taxonomy/templates/deployment.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Chart.Name }}-config + configMap: + name: {{ .Chart.Name }}-config + - name: {{ .Chart.Name }}-xml-config + configMap: + name: {{ .Chart.Name }}-xml-config + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + env: + - name: JAVA_OPTIONS + value: {{ .Values.env.javaoptions | quote }} + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Chart.Name }}-config + mountPath: /home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/application.conf + subPath: taxonomy-service_application.conf + - name: {{ .Chart.Name }}-xml-config + mountPath: /home/sunbird/taxonomy-service-1.0-SNAPSHOT/config/logback.xml + subPath: taxonomy-service_logback.xml + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/taxonomy/values.j2 b/kubernetes/helm_charts/core/taxonomy/values.j2 new file mode 100644 index 000000000..6079b4d2f --- /dev/null +++ b/kubernetes/helm_charts/core/taxonomy/values.j2 @@ -0,0 +1,28 @@ +### Default variable file for search-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +env: + javaoptions: {{taxonomy_java_mem_limit|default('-Xmx600m')}} + +replicaCount: {{taxonomy_replicacount|default(1)}} +repository: {{taxonomy_repository|default('taxonomy-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{taxonomy_cpu_req|default('100m')}} + memory: {{taxonomy_mem_req|default('100Mi')}} + limits: + cpu: {{taxonomy_cpu_limit|default('1')}} + memory: {{taxonomy_mem_limit|default('1024Mi')}} +network: + port: 9006 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ taxonomy_maxsurge|default('25%') }} + maxunavailable: {{ taxonomy_maxunavailable|default('25%') }} + +{{ taxonomy_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/telemetry-dp-logstash/Chart.yaml b/kubernetes/helm_charts/core/telemetry-dp-logstash/Chart.yaml new file mode 100644 index 000000000..3760b5476 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry-dp-logstash/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: telemetry-logstash-dock +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/telemetry-dp-logstash/templates/deployment.yaml b/kubernetes/helm_charts/core/telemetry-dp-logstash/templates/deployment.yaml new file mode 100644 index 000000000..6f584f7d2 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry-dp-logstash/templates/deployment.yaml @@ -0,0 +1,66 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ .Release.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + volumes: + - name: {{ .Release.Name }}-config + configMap: + name: {{ .Release.Name }}-config + containers: + - name: {{ .Release.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Release.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + volumeMounts: + - name: {{ .Release.Name }}-config + mountPath: /etc/telemetry-logstash.conf + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Release.Name }} +spec: + ports: + - name: http-{{ .Release.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/core/telemetry-dp-logstash/values.j2 b/kubernetes/helm_charts/core/telemetry-dp-logstash/values.j2 new file mode 100644 index 000000000..d6e104f35 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry-dp-logstash/values.j2 @@ -0,0 +1,23 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: sunbird + +replicaCount: {{telemetry_logstash_replicacount|default(1)}} +repository: 'telemetry_logstash' +image_tag: 5.6-alpine +resources: + requests: + cpu: {{telemetry_logstash_cpu_req|default('100m')}} + memory: {{telemetry_logstash_mem_req|default('100Mi')}} + limits: + cpu: {{telemetry_logstash_cpu_limit|default('1')}} + memory: {{telemetry_logstash_mem_limit|default('1024Mi')}} +network: + port: 5044 + targetport: 5044 +strategy: + type: RollingUpdate + maxsurge: {{ telemetry_logstash_maxsurge|default('25%') }} + maxunavailable: {{ telemetry_logstash_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/telemetry/.helmignore b/kubernetes/helm_charts/core/telemetry/.helmignore new file mode 100644 index 000000000..50af03172 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/core/telemetry/Chart.yaml b/kubernetes/helm_charts/core/telemetry/Chart.yaml new file mode 100644 index 000000000..e4f277fdf --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: telemetry +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/telemetry/templates/_helpers.tpl b/kubernetes/helm_charts/core/telemetry/templates/_helpers.tpl new file mode 100644 index 000000000..590205843 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/templates/_helpers.tpl @@ -0,0 +1,45 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "telemetry.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "telemetry.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "telemetry.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "telemetry.labels" -}} +app.kubernetes.io/name: {{ include "telemetry.name" . }} +helm.sh/chart: {{ include "telemetry.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/kubernetes/helm_charts/core/telemetry/templates/configmap.yaml b/kubernetes/helm_charts/core/telemetry/templates/configmap.yaml new file mode 100644 index 000000000..00fdbaa0c --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.telemetryenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/telemetry/templates/deployment.yaml b/kubernetes/helm_charts/core/telemetry/templates/deployment.yaml new file mode 100644 index 000000000..a82590540 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ .Chart.Name }}-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/telemetry/values.j2 b/kubernetes/helm_charts/core/telemetry/values.j2 new file mode 100644 index 000000000..e8dd079b5 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetry/values.j2 @@ -0,0 +1,26 @@ + +### variables for telemetry service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{telemetry_replicacount|default(1)}} +repository: {{telemetry_repository|default('telemetry-service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{telemetry_cpu_req|default('100m')}} + memory: {{telemetry_mem_req|default('100Mi')}} + limits: + cpu: {{telemetry_cpu_limit|default('1')}} + memory: {{telemetry_mem_limit|default('1024Mi')}} +network: + port: 9001 + targetport: 9001 +strategy: + type: RollingUpdate + maxsurge: {{ telemetry_maxsurge|default(1) }} + maxunavailable: {{ telemetry_maxunavailable|default }} + +{{ telemetry_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/core/telemetrydplogstash/values.j2 b/kubernetes/helm_charts/core/telemetrydplogstash/values.j2 new file mode 100644 index 000000000..d6e104f35 --- /dev/null +++ b/kubernetes/helm_charts/core/telemetrydplogstash/values.j2 @@ -0,0 +1,23 @@ +### Default variable file for cert-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: sunbird + +replicaCount: {{telemetry_logstash_replicacount|default(1)}} +repository: 'telemetry_logstash' +image_tag: 5.6-alpine +resources: + requests: + cpu: {{telemetry_logstash_cpu_req|default('100m')}} + memory: {{telemetry_logstash_mem_req|default('100Mi')}} + limits: + cpu: {{telemetry_logstash_cpu_limit|default('1')}} + memory: {{telemetry_logstash_mem_limit|default('1024Mi')}} +network: + port: 5044 + targetport: 5044 +strategy: + type: RollingUpdate + maxsurge: {{ telemetry_logstash_maxsurge|default('25%') }} + maxunavailable: {{ telemetry_logstash_maxunavailable|default('25%') }} diff --git a/kubernetes/helm_charts/core/userorg/Chart.yaml b/kubernetes/helm_charts/core/userorg/Chart.yaml new file mode 100644 index 000000000..c7c993ab6 --- /dev/null +++ b/kubernetes/helm_charts/core/userorg/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +appVersion: "1.0" +description: A Helm chart for Kubernetes +name: userorg +version: 0.1.0 diff --git a/kubernetes/helm_charts/core/userorg/templates/configmap.yaml b/kubernetes/helm_charts/core/userorg/templates/configmap.yaml new file mode 100644 index 000000000..94fd443c7 --- /dev/null +++ b/kubernetes/helm_charts/core/userorg/templates/configmap.yaml @@ -0,0 +1,10 @@ +#apiVersion: v1 +#data: +# {{- range $key, $val := .Values.userorgenv }} +# {{ $key }}: {{ $val }} +# {{- end }} +#kind: ConfigMap +#metadata: +# creationTimestamp: null +# name: {{ .Chart.Name }}-config +# namespace: {{ .Values.namespace }} diff --git a/kubernetes/helm_charts/core/userorg/templates/deployment.yaml b/kubernetes/helm_charts/core/userorg/templates/deployment.yaml new file mode 100644 index 000000000..d3f83edaf --- /dev/null +++ b/kubernetes/helm_charts/core/userorg/templates/deployment.yaml @@ -0,0 +1,59 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Chart.Name }} + namespace: {{ .Values.namespace }} + annotations: + reloader.stakater.com/auto: "true" +spec: + replicas: {{ .Values.replicaCount }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.strategy.maxsurge }} + maxUnavailable: {{ .Values.strategy.maxunavailable }} + selector: + matchLabels: + app: {{ .Chart.Name }} + template: + metadata: + labels: + app: {{ .Chart.Name }} + spec: +{{- if .Values.imagepullsecrets }} + imagePullSecrets: + - name: {{ .Values.imagepullsecrets }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}" + imagePullPolicy: Always + envFrom: + - configMapRef: + name: {{ .Chart.Name }}-config + resources: +{{ toYaml .Values.resources | indent 10 }} + ports: + - containerPort: {{ .Values.network.port }} + {{- if .Values.healthcheck }} + livenessProbe: +{{ toYaml .Values.livenessProbe | indent 10 }} + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + {{- end }} + +--- +apiVersion: v1 +kind: Service +metadata: + name: user-org-service + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} +spec: + ports: + - name: http-{{ .Chart.Name }} + protocol: TCP + port: {{ .Values.network.targetport }} + selector: + app: {{ .Chart.Name }} diff --git a/kubernetes/helm_charts/core/userorg/values.j2 b/kubernetes/helm_charts/core/userorg/values.j2 new file mode 100644 index 000000000..6508b4d13 --- /dev/null +++ b/kubernetes/helm_charts/core/userorg/values.j2 @@ -0,0 +1,25 @@ +### Default variable file for userorg-service ### + +namespace: {{ namespace }} +imagepullsecrets: {{ imagepullsecrets }} +dockerhub: {{ dockerhub }} + +replicaCount: {{userorg_replicacount|default(1)}} +repository: {{userorg_repository|default('user_org_service')}} +image_tag: {{ image_tag }} +resources: + requests: + cpu: {{userorg_cpu_req|default('100m')}} + memory: {{userorg_mem_req|default('100Mi')}} + limits: + cpu: {{userorg_cpu_limit|default('1')}} + memory: {{userorg_mem_limit|default('1024Mi')}} +network: + port: 9008 + targetport: 9000 +strategy: + type: RollingUpdate + maxsurge: {{ userorg_maxsurge|default('25%') }} + maxunavailable: {{ userorg_maxunavailable|default('25%') }} + +{{ userorg_liveness_readiness | to_nice_yaml }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/Chart.yaml b/kubernetes/helm_charts/istio-system/istio-init/Chart.yaml new file mode 100755 index 000000000..2dd6de9d9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart to initialize Istio CRDs +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- crd +name: istio-init +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2-0' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio-init/README.md b/kubernetes/helm_charts/istio-system/istio-init/README.md new file mode 100755 index 000000000..c0a0e34f8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/README.md @@ -0,0 +1,77 @@ +# Istio + +[Istio](https://istio.io/) is an open platform for providing a uniform way to integrate microservices, manage traffic flow across microservices, enforce policies and aggregate telemetry data. + +## Introduction + +This chart bootstraps Istio's [CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#customresourcedefinitions) +which are an internal implementation detail of Istio. CRDs define data structures for storing runtime configuration +specified by a human operator. + +This chart must be run to completion prior to running other Istio charts, or other Istio charts will fail to initialize. + +## Prerequisites + +- Kubernetes 1.9 or newer cluster with RBAC (Role-Based Access Control) enabled is required +- Helm 2.7.2 or newer or alternately the ability to modify RBAC rules is also required + +## Resources Required + +The chart deploys pods that consume minimal resources. + +## Installing the Chart + +1. If a service account has not already been installed for Tiller, install one: + ``` + $ kubectl apply -f install/kubernetes/helm/helm-service-account.yaml + ``` + +1. If Tiller has not already been installed in your cluster, Install Tiller on your cluster with the service account: + ``` + $ helm init --service-account tiller + ``` + +1. Install the Istio initializer chart: + ``` + $ helm install install/kubernetes/helm/istio-init --name istio-init --namespace istio-system + ``` + + > Although you can install the `istio-init` chart to any namespace, it is recommended to install `istio-init` in the same namespace(`istio-system`) as other Istio charts. + +## Configuration + +The Helm chart ships with reasonable defaults. There may be circumstances in which defaults require overrides. +To override Helm values, use `--set key=value` argument during the `helm install` command. Multiple `--set` operations may be used in the same Helm operation. + +Helm charts expose configuration options which are currently in alpha. The currently exposed options are explained in the following table: + +| Parameter | Description | Values | Default | +| --- | --- | --- | --- | +| `global.hub` | Specifies the HUB for most images used by Istio | registry/namespace | `docker.io/istio` | +| `global.tag` | Specifies the TAG for most images used by Istio | valid image tag | `0.8.latest` | +| `global.imagePullPolicy` | Specifies the image pull policy | valid image pull policy | `IfNotPresent` | + + +## Uninstalling the Chart + +> Uninstalling this chart does not delete Istio's registered CRDs. Istio by design expects +> CRDs to leak into the Kubernetes environment. As CRDs contain all runtime configuration +> data in CustomResources the Istio designers feel it is better to explicitly delete this +> configuration rather then unexpectedly lose it. + +To uninstall/delete the `istio-init` release but continue to track the release: + ``` + $ helm delete istio-init + ``` + +To uninstall/delete the `istio-init` release completely and make its name free for later use: + ``` + $ helm delete --purge istio-init + ``` + +> Warning: Deleting CRDs will delete any configuration that you have made to Istio. + +To delete all CRDs, run the following command + ``` + $ for i in istio-init/files/*crd*yaml; do kubectl delete -f $i; done + ``` diff --git a/kubernetes/helm_charts/istio-system/istio-init/files/crd-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/files/crd-10.yaml new file mode 100755 index 000000000..e76e50edc --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/files/crd-10.yaml @@ -0,0 +1,636 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: virtualservices.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: VirtualService + listKind: VirtualServiceList + plural: virtualservices + singular: virtualservice + shortNames: + - vs + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true + additionalPrinterColumns: + - JSONPath: .spec.gateways + description: The names of gateways and sidecars that should apply these routes + name: Gateways + type: string + - JSONPath: .spec.hosts + description: The destination hosts to which traffic is being sent + name: Hosts + type: string + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: destinationrules.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: DestinationRule + listKind: DestinationRuleList + plural: destinationrules + singular: destinationrule + shortNames: + - dr + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true + additionalPrinterColumns: + - JSONPath: .spec.host + description: The name of a service from the service registry + name: Host + type: string + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: serviceentries.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: ServiceEntry + listKind: ServiceEntryList + plural: serviceentries + singular: serviceentry + shortNames: + - se + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true + additionalPrinterColumns: + - JSONPath: .spec.hosts + description: The hosts associated with the ServiceEntry + name: Hosts + type: string + - JSONPath: .spec.location + description: Whether the service is external to the mesh or part of the mesh (MESH_EXTERNAL or MESH_INTERNAL) + name: Location + type: string + - JSONPath: .spec.resolution + description: Service discovery mode for the hosts (NONE, STATIC, or DNS) + name: Resolution + type: string + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: gateways.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: Gateway + plural: gateways + singular: gateway + shortNames: + - gw + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: envoyfilters.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: EnvoyFilter + plural: envoyfilters + singular: envoyfilter + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: clusterrbacconfigs.rbac.istio.io + labels: + app: istio-pilot + istio: rbac + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: rbac.istio.io + names: + kind: ClusterRbacConfig + plural: clusterrbacconfigs + singular: clusterrbacconfig + categories: + - istio-io + - rbac-istio-io + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: policies.authentication.istio.io + labels: + app: istio-citadel + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: authentication.istio.io + names: + kind: Policy + plural: policies + singular: policy + categories: + - istio-io + - authentication-istio-io + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: meshpolicies.authentication.istio.io + labels: + app: istio-citadel + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: authentication.istio.io + names: + kind: MeshPolicy + listKind: MeshPolicyList + plural: meshpolicies + singular: meshpolicy + categories: + - istio-io + - authentication-istio-io + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: httpapispecbindings.config.istio.io + labels: + app: istio-mixer + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: HTTPAPISpecBinding + plural: httpapispecbindings + singular: httpapispecbinding + categories: + - istio-io + - apim-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: httpapispecs.config.istio.io + labels: + app: istio-mixer + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: HTTPAPISpec + plural: httpapispecs + singular: httpapispec + categories: + - istio-io + - apim-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: quotaspecbindings.config.istio.io + labels: + app: istio-mixer + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: QuotaSpecBinding + plural: quotaspecbindings + singular: quotaspecbinding + categories: + - istio-io + - apim-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: quotaspecs.config.istio.io + labels: + app: istio-mixer + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: QuotaSpec + plural: quotaspecs + singular: quotaspec + categories: + - istio-io + - apim-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: rules.config.istio.io + labels: + app: mixer + package: istio.io.mixer + istio: core + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: rule + plural: rules + singular: rule + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: attributemanifests.config.istio.io + labels: + app: mixer + package: istio.io.mixer + istio: core + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: attributemanifest + plural: attributemanifests + singular: attributemanifest + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: rbacconfigs.rbac.istio.io + labels: + app: mixer + package: istio.io.mixer + istio: rbac + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: rbac.istio.io + names: + kind: RbacConfig + plural: rbacconfigs + singular: rbacconfig + categories: + - istio-io + - rbac-istio-io + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: serviceroles.rbac.istio.io + labels: + app: mixer + package: istio.io.mixer + istio: rbac + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: rbac.istio.io + names: + kind: ServiceRole + plural: serviceroles + singular: servicerole + categories: + - istio-io + - rbac-istio-io + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: servicerolebindings.rbac.istio.io + labels: + app: mixer + package: istio.io.mixer + istio: rbac + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: rbac.istio.io + names: + kind: ServiceRoleBinding + plural: servicerolebindings + singular: servicerolebinding + categories: + - istio-io + - rbac-istio-io + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + additionalPrinterColumns: + - JSONPath: .spec.roleRef.name + description: The name of the ServiceRole object being referenced + name: Reference + type: string + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: adapters.config.istio.io + labels: + app: mixer + package: adapter + istio: mixer-adapter + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: adapter + plural: adapters + singular: adapter + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: instances.config.istio.io + labels: + app: mixer + package: instance + istio: mixer-instance + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: instance + plural: instances + singular: instance + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: templates.config.istio.io + labels: + app: mixer + package: template + istio: mixer-template + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: template + plural: templates + singular: template + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: handlers.config.istio.io + labels: + app: mixer + package: handler + istio: mixer-handler + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: config.istio.io + names: + kind: handler + plural: handlers + singular: handler + categories: + - istio-io + - policy-istio-io + scope: Namespaced + versions: + - name: v1alpha2 + served: true + storage: true +--- diff --git a/kubernetes/helm_charts/istio-system/istio-init/files/crd-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/files/crd-11.yaml new file mode 100755 index 000000000..5087d38fa --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/files/crd-11.yaml @@ -0,0 +1,26 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: sidecars.networking.istio.io + labels: + app: istio-pilot + chart: istio + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: networking.istio.io + names: + kind: Sidecar + plural: sidecars + singular: sidecar + categories: + - istio-io + - networking-istio-io + scope: Namespaced + versions: + - name: v1alpha3 + served: true + storage: true +--- diff --git a/kubernetes/helm_charts/istio-system/istio-init/files/crd-12.yaml b/kubernetes/helm_charts/istio-system/istio-init/files/crd-12.yaml new file mode 100755 index 000000000..d9b337244 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/files/crd-12.yaml @@ -0,0 +1,24 @@ +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: authorizationpolicies.rbac.istio.io + labels: + app: istio-pilot + istio: rbac + heritage: Tiller + release: istio +spec: + group: rbac.istio.io + names: + kind: AuthorizationPolicy + plural: authorizationpolicies + singular: authorizationpolicy + categories: + - istio-io + - rbac-istio-io + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true +--- diff --git a/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-10.yaml new file mode 100755 index 000000000..d3a3069e8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-10.yaml @@ -0,0 +1,91 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: clusterissuers.certmanager.k8s.io + labels: + app: certmanager + chart: certmanager + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: certmanager.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: ClusterIssuer + plural: clusterissuers + scope: Cluster +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: issuers.certmanager.k8s.io + labels: + app: certmanager + chart: certmanager + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + group: certmanager.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: Issuer + plural: issuers + scope: Namespaced +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: certificates.certmanager.k8s.io + labels: + app: certmanager + chart: certmanager + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + additionalPrinterColumns: + - JSONPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - JSONPath: .spec.secretName + name: Secret + type: string + - JSONPath: .spec.issuerRef.name + name: Issuer + type: string + priority: 1 + - JSONPath: .status.conditions[?(@.type=="Ready")].message + name: Status + type: string + priority: 1 + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date + group: certmanager.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + scope: Namespaced + names: + kind: Certificate + plural: certificates + shortNames: + - cert + - certs +--- diff --git a/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-11.yaml new file mode 100755 index 000000000..f63787b68 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/files/crd-certmanager-11.yaml @@ -0,0 +1,80 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: orders.certmanager.k8s.io + labels: + app: certmanager + chart: certmanager + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + additionalPrinterColumns: + - JSONPath: .status.state + name: State + type: string + - JSONPath: .spec.issuerRef.name + name: Issuer + type: string + priority: 1 + - JSONPath: .status.reason + name: Reason + type: string + priority: 1 + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date + group: certmanager.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: Order + plural: orders + scope: Namespaced +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: challenges.certmanager.k8s.io + labels: + app: certmanager + chart: certmanager + heritage: Tiller + release: istio + annotations: + "helm.sh/resource-policy": keep +spec: + additionalPrinterColumns: + - JSONPath: .status.state + name: State + type: string + - JSONPath: .spec.dnsName + name: Domain + type: string + - JSONPath: .status.reason + name: Reason + type: string + - JSONPath: .metadata.creationTimestamp + description: |- + CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. + + Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata + name: Age + type: date + group: certmanager.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + names: + kind: Challenge + plural: challenges + scope: Namespaced +--- diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrole.yaml new file mode 100755 index 000000000..0b7c50fbc --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrole.yaml @@ -0,0 +1,11 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-init-{{ .Release.Namespace }} + labels: + app: istio-init + istio: init +rules: +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["create", "get", "list", "watch", "patch"] diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..481674c0e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/clusterrolebinding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-init-admin-role-binding-{{ .Release.Namespace }} + labels: + app: istio-init + istio: init +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-init-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-init-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-10.yaml new file mode 100755 index 000000000..69e37fa14 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-10.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Release.Namespace }} + name: istio-crd-10 +data: + crd-10.yaml: |- +{{.Files.Get "files/crd-10.yaml" | printf "%s" | indent 4}} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-11.yaml new file mode 100755 index 000000000..952640d60 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-11.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Release.Namespace }} + name: istio-crd-11 +data: + crd-11.yaml: |- +{{.Files.Get "files/crd-11.yaml" | printf "%s" | indent 4}} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-12.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-12.yaml new file mode 100755 index 000000000..a49736534 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-12.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Release.Namespace }} + name: istio-crd-12 +data: + crd-12.yaml: |- +{{.Files.Get "files/crd-12.yaml" | printf "%s" | indent 4}} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-10.yaml new file mode 100755 index 000000000..8ab3e8356 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-10.yaml @@ -0,0 +1,10 @@ +{{- if .Values.certmanager.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Release.Namespace }} + name: istio-crd-certmanager-10 +data: + crd-certmanager-10.yaml: |- +{{.Files.Get "files/crd-certmanager-10.yaml" | printf "%s" | indent 4}} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-11.yaml new file mode 100755 index 000000000..beef3043d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/configmap-crd-certmanager-11.yaml @@ -0,0 +1,10 @@ +{{- if .Values.certmanager.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: {{ .Release.Namespace }} + name: istio-crd-certmanager-11 +data: + crd-certmanager-11.yaml: |- +{{.Files.Get "files/crd-certmanager-11.yaml" | printf "%s" | indent 4}} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-10.yaml new file mode 100755 index 000000000..2f98bc51f --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-10.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ .Release.Namespace }} + name: istio-init-crd-10-{{ .Values.global.tag | printf "%v" | trunc 32 }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-init-service-account + containers: + - name: istio-init-crd-10 + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - name: crd-10 + mountPath: /etc/istio/crd-10 + readOnly: true + command: ["kubectl", "apply", "-f", "/etc/istio/crd-10/crd-10.yaml"] + volumes: + - name: crd-10 + configMap: + name: istio-crd-10 + restartPolicy: OnFailure diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-11.yaml new file mode 100755 index 000000000..35996a022 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-11.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ .Release.Namespace }} + name: istio-init-crd-11-{{ .Values.global.tag | printf "%v" | trunc 32 }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-init-service-account + containers: + - name: istio-init-crd-11 + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - name: crd-11 + mountPath: /etc/istio/crd-11 + readOnly: true + command: ["kubectl", "apply", "-f", "/etc/istio/crd-11/crd-11.yaml"] + volumes: + - name: crd-11 + configMap: + name: istio-crd-11 + restartPolicy: OnFailure diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-12.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-12.yaml new file mode 100755 index 000000000..f40425097 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-12.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ .Release.Namespace }} + name: istio-init-crd-12-{{ .Values.global.tag | printf "%v" | trunc 32 }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-init-service-account + containers: + - name: istio-init-crd-12 + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - name: crd-12 + mountPath: /etc/istio/crd-12 + readOnly: true + command: ["kubectl", "apply", "-f", "/etc/istio/crd-12/crd-12.yaml"] + volumes: + - name: crd-12 + configMap: + name: istio-crd-12 + restartPolicy: OnFailure diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-10.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-10.yaml new file mode 100755 index 000000000..5b4e0a268 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-10.yaml @@ -0,0 +1,28 @@ +{{- if .Values.certmanager.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ .Release.Namespace }} + name: istio-init-crd-certmanager-10-{{ .Values.global.tag | printf "%v" | trunc 32 }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-init-service-account + containers: + - name: istio-init-crd-certmanager-10 + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - name: crd-certmanager-10 + mountPath: /etc/istio/crd-certmanager-10 + readOnly: true + command: ["kubectl", "apply", "-f", "/etc/istio/crd-certmanager-10/crd-certmanager-10.yaml"] + volumes: + - name: crd-certmanager-10 + configMap: + name: istio-crd-certmanager-10 + restartPolicy: OnFailure +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-11.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-11.yaml new file mode 100755 index 000000000..06092cf58 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/job-crd-certmanager-11.yaml @@ -0,0 +1,28 @@ +{{- if .Values.certmanager.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + namespace: {{ .Release.Namespace }} + name: istio-init-crd-certmanager-11-{{ .Values.global.tag | printf "%v" | trunc 32 }} +spec: + template: + metadata: + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-init-service-account + containers: + - name: istio-init-crd-certmanager-11 + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - name: crd-certmanager-11 + mountPath: /etc/istio/crd-certmanager-11 + readOnly: true + command: ["kubectl", "apply", "-f", "/etc/istio/crd-certmanager-11/crd-certmanager-11.yaml"] + volumes: + - name: crd-certmanager-11 + configMap: + name: istio-crd-certmanager-11 + restartPolicy: OnFailure +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio-init/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio-init/templates/serviceaccount.yaml new file mode 100755 index 000000000..314666293 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-init-service-account + namespace: {{ .Release.Namespace }} + labels: + app: istio-init + istio: init + diff --git a/kubernetes/helm_charts/istio-system/istio-init/values.yaml b/kubernetes/helm_charts/istio-system/istio-init/values.yaml new file mode 100755 index 000000000..6a562cb32 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio-init/values.yaml @@ -0,0 +1,16 @@ +global: + # Default hub for Istio images. + # Releases are published to docker hub under 'istio' project. + # Daily builds from prow are on gcr.io + hub: docker.io/istio + + # Default tag for Istio images. + tag: 1.3.3 + + # imagePullPolicy is applied to istio control plane components. + # local tests require IfNotPresent, to avoid uploading to dockerhub. + # TODO: Switch to Always as default, and override in the local tests. + imagePullPolicy: IfNotPresent + +certmanager: + enabled: false diff --git a/kubernetes/helm_charts/istio-system/istio/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/Chart.yaml new file mode 100755 index 000000000..1b02a34bd --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/Chart.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for all istio components +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- security +- sidecarInjectorWebhook +- mixer +- pilot +- galley +name: istio +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2-0' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/README.md b/kubernetes/helm_charts/istio-system/istio/README.md new file mode 100755 index 000000000..de67ba20b --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/README.md @@ -0,0 +1,140 @@ +# Istio + +[Istio](https://istio.io/) is an open platform for providing a uniform way to integrate microservices, manage traffic flow across microservices, enforce policies and aggregate telemetry data. + + + +The documentation here is for developers only, please follow the installation instructions from [istio.io](https://istio.io/docs/setup/kubernetes/install/helm/) for all other uses. + +## Introduction + +This chart bootstraps all Istio [components](https://istio.io/docs/concepts/what-is-istio/) deployment on a [Kubernetes](http://kubernetes.io) cluster using the [Helm](https://helm.sh) package manager. + +## Chart Details + +This chart can install multiple Istio components as subcharts: +- ingressgateway +- egressgateway +- sidecarInjectorWebhook +- galley +- mixer +- pilot +- security(citadel) +- grafana +- prometheus +- tracing(jaeger) +- kiali + +To enable or disable each component, change the corresponding `enabled` flag. + +## Prerequisites + +- Kubernetes 1.9 or newer cluster with RBAC (Role-Based Access Control) enabled is required +- Helm 2.7.2 or newer or alternately the ability to modify RBAC rules is also required +- If you want to enable automatic sidecar injection, Kubernetes 1.9+ with `admissionregistration` API is required, and `kube-apiserver` process must have the `admission-control` flag set with the `MutatingAdmissionWebhook` and `ValidatingAdmissionWebhook` admission controllers added and listed in the correct order. +- The `istio-init` chart must be run to completion prior to install the `istio` chart. + +## Resources Required + +The chart deploys pods that consume minimum resources as specified in the resources configuration parameter. + +## Installing the Chart + +1. If a service account has not already been installed for Tiller, install one: + ``` + $ kubectl apply -f install/kubernetes/helm/helm-service-account.yaml + ``` + +1. Install Tiller on your cluster with the service account: + ``` + $ helm init --service-account tiller + ``` + +1. Set and create the namespace where Istio was installed: + ``` + $ NAMESPACE=istio-system + $ kubectl create ns $NAMESPACE + ``` + +1. If you are enabling `kiali`, you need to create the secret that contains the username and passphrase for `kiali` dashboard: + ``` + $ echo -n 'admin' | base64 + YWRtaW4= + $ echo -n '1f2d1e2e67df' | base64 + MWYyZDFlMmU2N2Rm + $ cat <=1.9.0): + ``` + $ helm install istio --name istio --namespace $NAMESPACE + ``` + + - Without the sidecar injection webhook: + ``` + $ helm install istio --name istio --namespace $NAMESPACE --set sidecarInjectorWebhook.enabled=false + ``` + +## Configuration + +The Helm chart ships with reasonable defaults. There may be circumstances in which defaults require overrides. +To override Helm values, use `--set key=value` argument during the `helm install` command. Multiple `--set` operations may be used in the same Helm operation. + +Helm charts expose configuration options which are currently in alpha. The currently exposed options can be found [here](https://istio.io/docs/reference/config/installation-options/). + +## Uninstalling the Chart + +To uninstall/delete the `istio` release but continue to track the release: + ``` + $ helm delete istio + ``` + +To uninstall/delete the `istio` release completely and make its name free for later use: + ``` + $ helm delete --purge istio + ``` diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/Chart.yaml new file mode 100755 index 000000000..800e40e73 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +appVersion: 0.6.2 +description: A Helm chart for Kubernetes +name: certmanager +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/NOTES.txt b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/NOTES.txt new file mode 100755 index 000000000..0307ede4c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/NOTES.txt @@ -0,0 +1,6 @@ +certmanager has been deployed successfully! + +More information on the different types of issuers and how to configure them +can be found in our documentation: + +https://cert-manager.readthedocs.io/en/latest/reference/issuers.html \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/_helpers.tpl new file mode 100755 index 000000000..331a91d43 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "certmanager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "certmanager.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "certmanager.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/deployment.yaml new file mode 100755 index 000000000..48e473153 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/deployment.yaml @@ -0,0 +1,69 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: certmanager + namespace: {{ .Release.Namespace }} + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: certmanager + template: + metadata: + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 8 }} + {{- end }} + annotations: + sidecar.istio.io/inject: "false" + {{- if .Values.podAnnotations }} +{{ toYaml .Values.podAnnotations | indent 8 }} + {{- end }} + spec: + serviceAccountName: certmanager +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: certmanager + image: "{{ .Values.hub }}/{{ .Values.image }}:{{ .Values.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + - --cluster-resource-namespace=$(POD_NAMESPACE) + - --leader-election-namespace=$(POD_NAMESPACE) + {{- if .Values.extraArgs }} +{{ toYaml .Values.extraArgs | indent 8 }} + {{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: +{{ toYaml .Values.resources | indent 10 }} + {{- if .Values.podDnsPolicy }} + dnsPolicy: {{ .Values.podDnsPolicy }} + {{- end }} + {{- if .Values.podDnsConfig }} + dnsConfig: +{{ toYaml .Values.podDnsConfig | indent 8 }} + {{- end }} + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} +{{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/issuer.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/issuer.yaml new file mode 100755 index 000000000..59402daea --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/issuer.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging + namespace: {{ .Release.Namespace }} + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: {{ .Values.email }} + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-staging + http01: {} +--- +apiVersion: certmanager.k8s.io/v1alpha1 +kind: ClusterIssuer +metadata: + name: letsencrypt + namespace: {{ .Release.Namespace }} + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: {{ .Values.email }} + privateKeySecretRef: + name: letsencrypt + http01: {} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..b251e3653 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/poddisruptionbudget.yaml @@ -0,0 +1,24 @@ +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: certmanager + namespace: {{ .Release.Namespace }} + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + version: {{ .Chart.Version }} + {{- if .Values.podLabels }} +{{ toYaml .Values.podLabels | indent 4 }} + {{- end }} +spec: +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +{{ include "podDisruptionBudget.spec" .Values.global.defaultPodDisruptionBudget }} +{{- end }} + selector: + matchLabels: + app: certmanager + release: {{ .Release.Name }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/rbac.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/rbac.yaml new file mode 100755 index 000000000..b3a4ef340 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/rbac.yaml @@ -0,0 +1,37 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: certmanager + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: + - apiGroups: ["certmanager.k8s.io"] + resources: ["certificates", "certificates/finalizers", "issuers", "clusterissuers", "orders", "orders/finalizers", "challenges"] + verbs: ["*"] + - apiGroups: [""] + resources: ["configmaps", "secrets", "events", "services", "pods"] + verbs: ["*"] + - apiGroups: ["extensions"] + resources: ["ingresses"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: certmanager + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: certmanager +subjects: + - name: certmanager + namespace: {{ .Release.Namespace }} + kind: ServiceAccount diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/serviceaccount.yaml new file mode 100755 index 000000000..f87543508 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: certmanager + namespace: {{ .Release.Namespace }} + labels: + app: certmanager + chart: {{ template "certmanager.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/certmanager/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/values.yaml new file mode 100755 index 000000000..775cbeb1e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/certmanager/values.yaml @@ -0,0 +1,34 @@ +# Certmanager uses ACME to sign certificates. Since Istio gateways are +# mounting the TLS secrets the Certificate CRDs must be created in the +# istio-system namespace. Once the certificate has been created, the +# gateway must be updated by adding 'secretVolumes'. After the gateway +# restart, DestinationRules can be created using the ACME-signed certificates. +enabled: false +replicaCount: 1 +hub: quay.io/jetstack +image: cert-manager-controller +tag: v0.6.2 +resources: {} +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/Chart.yaml new file mode 100755 index 000000000..68f0a5c1b --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for galley deployment +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- galley +name: galley +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/_helpers.tpl new file mode 100755 index 000000000..5d42f4a03 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "galley.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "galley.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "galley.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrole.yaml new file mode 100755 index 000000000..8abc797ac --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrole.yaml @@ -0,0 +1,42 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-galley-{{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["*"] +- apiGroups: ["config.istio.io"] # istio mixer CRD watcher + resources: ["*"] + verbs: ["get", "list", "watch"] +- apiGroups: ["networking.istio.io"] + resources: ["*"] + verbs: ["get", "list", "watch"] +- apiGroups: ["authentication.istio.io"] + resources: ["*"] + verbs: ["get", "list", "watch"] +- apiGroups: ["rbac.istio.io"] + resources: ["*"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions","apps"] + resources: ["deployments"] + resourceNames: ["istio-galley"] + verbs: ["get"] +- apiGroups: [""] + resources: ["pods", "nodes", "services", "endpoints", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: ["ingresses"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions"] + resources: ["deployments/finalizers"] + resourceNames: ["istio-galley"] + verbs: ["update"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..88cde2554 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-galley-admin-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-galley-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-galley-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/configmap.yaml new file mode 100755 index 000000000..662c960b1 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/configmap.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-galley-configuration + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley +data: +{{- if .Values.global.configValidation }} + validatingwebhookconfiguration.yaml: |- + {{- include "validatingwebhookconfiguration.yaml.tpl" . | indent 4}} +{{- end}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/deployment.yaml new file mode 100755 index 000000000..af417ee8e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/deployment.yaml @@ -0,0 +1,127 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-galley + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + istio: galley + strategy: + rollingUpdate: + maxSurge: {{ .Values.rollingMaxSurge }} + maxUnavailable: {{ .Values.rollingMaxUnavailable }} + template: + metadata: + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-galley-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: galley +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: 443 + - containerPort: {{ .Values.global.monitoringPort }} + - containerPort: 9901 + command: + - /usr/local/bin/galley + - server + - --meshConfigFile=/etc/mesh-config/mesh + - --livenessProbeInterval=1s + - --livenessProbePath=/healthliveness + - --readinessProbePath=/healthready + - --readinessProbeInterval=1s + - --deployment-namespace={{ .Release.Namespace }} +{{- if $.Values.global.controlPlaneSecurityEnabled}} + - --insecure=false +{{- else }} + - --insecure=true +{{- end }} +{{- if not $.Values.global.useMCP }} + - --enable-server=false +{{- end }} +{{- if not $.Values.global.configValidation }} + - --enable-validation=false +{{- end }} + - --validation-webhook-config-file + - /etc/config/validatingwebhookconfiguration.yaml + - --monitoringPort={{ .Values.global.monitoringPort }} +{{- if $.Values.global.logging.level }} + - --log_output_level={{ $.Values.global.logging.level }} +{{- end}} + volumeMounts: + - name: certs + mountPath: /etc/certs + readOnly: true + - name: config + mountPath: /etc/config + readOnly: true + - name: mesh-config + mountPath: /etc/mesh-config + readOnly: true + livenessProbe: + exec: + command: + - /usr/local/bin/galley + - probe + - --probe-path=/healthliveness + - --interval=10s + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + exec: + command: + - /usr/local/bin/galley + - probe + - --probe-path=/healthready + - --interval=10s + initialDelaySeconds: 5 + periodSeconds: 5 + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumes: + - name: certs + secret: + secretName: istio.istio-galley-service-account + - name: config + configMap: + name: istio-galley-configuration + - name: mesh-config + configMap: + name: istio + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..75bf77834 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: istio-galley + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley +spec: +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +{{ include "podDisruptionBudget.spec" .Values.global.defaultPodDisruptionBudget }} +{{- end }} + selector: + matchLabels: + app: {{ template "galley.name" . }} + release: {{ .Release.Name }} + istio: galley +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/service.yaml new file mode 100755 index 000000000..cd21fd192 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/service.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-galley + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley +spec: + ports: + - port: 443 + name: https-validation + - port: {{ .Values.global.monitoringPort }} + name: http-monitoring + - port: 9901 + name: grpc-mcp + selector: + istio: galley diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/serviceaccount.yaml new file mode 100755 index 000000000..1ff54c49e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-galley-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/validatingwebhookconfiguration.yaml.tpl b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/validatingwebhookconfiguration.yaml.tpl new file mode 100755 index 000000000..ce68fb8a5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/templates/validatingwebhookconfiguration.yaml.tpl @@ -0,0 +1,118 @@ +{{ define "validatingwebhookconfiguration.yaml.tpl" }} +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: ValidatingWebhookConfiguration +metadata: + name: istio-galley + labels: + app: {{ template "galley.name" . }} + chart: {{ template "galley.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: galley +webhooks: + - name: pilot.validation.istio.io + clientConfig: + service: + name: istio-galley + namespace: {{ .Release.Namespace }} + path: "/admitpilot" + caBundle: "" + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - config.istio.io + apiVersions: + - v1alpha2 + resources: + - httpapispecs + - httpapispecbindings + - quotaspecs + - quotaspecbindings + - operations: + - CREATE + - UPDATE + apiGroups: + - rbac.istio.io + apiVersions: + - "*" + resources: + - "*" + - operations: + - CREATE + - UPDATE + apiGroups: + - authentication.istio.io + apiVersions: + - "*" + resources: + - "*" + - operations: + - CREATE + - UPDATE + apiGroups: + - networking.istio.io + apiVersions: + - "*" + resources: + - destinationrules + - envoyfilters + - gateways + - serviceentries + - sidecars + - virtualservices + failurePolicy: Fail + sideEffects: None + - name: mixer.validation.istio.io + clientConfig: + service: + name: istio-galley + namespace: {{ .Release.Namespace }} + path: "/admitmixer" + caBundle: "" + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - config.istio.io + apiVersions: + - v1alpha2 + resources: + - rules + - attributemanifests + - circonuses + - deniers + - fluentds + - kubernetesenvs + - listcheckers + - memquotas + - noops + - opas + - prometheuses + - rbacs + - solarwindses + - stackdrivers + - cloudwatches + - dogstatsds + - statsds + - stdios + - apikeys + - authorizations + - checknothings + # - kuberneteses + - listentries + - logentries + - metrics + - quotas + - reportnothings + - tracespans + - adapters + - handlers + - instances + - templates + - zipkins + failurePolicy: Fail + sideEffects: None +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/galley/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/galley/values.yaml new file mode 100755 index 000000000..a1d3a8e77 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/galley/values.yaml @@ -0,0 +1,31 @@ +# +# galley configuration +# +enabled: true +replicaCount: 1 +rollingMaxSurge: 100% +rollingMaxUnavailable: 25% +image: galley +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/Chart.yaml new file mode 100755 index 000000000..b49a15330 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for deploying Istio gateways +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- ingressgateway +- egressgateway +- gateways +name: gateways +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_affinity.tpl b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_affinity.tpl new file mode 100755 index 000000000..fbd0e9a8f --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_affinity.tpl @@ -0,0 +1,93 @@ +{{/* affinity - https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ */}} + +{{- define "gatewaynodeaffinity" }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "gatewayNodeAffinityRequiredDuringScheduling" . }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "gatewayNodeAffinityPreferredDuringScheduling" . }} +{{- end }} + +{{- define "gatewayNodeAffinityRequiredDuringScheduling" }} + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + {{- range $key, $val := .root.Values.global.arch }} + {{- if gt ($val | int) 0 }} + - {{ $key | quote }} + {{- end }} + {{- end }} + {{- $nodeSelector := default .root.Values.global.defaultNodeSelector .nodeSelector -}} + {{- range $key, $val := $nodeSelector }} + - key: {{ $key }} + operator: In + values: + - {{ $val | quote }} + {{- end }} +{{- end }} + +{{- define "gatewayNodeAffinityPreferredDuringScheduling" }} + {{- range $key, $val := .root.Values.global.arch }} + {{- if gt ($val | int) 0 }} + - weight: {{ $val | int }} + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - {{ $key | quote }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "gatewaypodAntiAffinity" }} +{{- if or .podAntiAffinityLabelSelector .podAntiAffinityTermLabelSelector}} + podAntiAffinity: + {{- if .podAntiAffinityLabelSelector }} + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "gatewaypodAntiAffinityRequiredDuringScheduling" . }} + {{- end }} + {{- if .podAntiAffinityTermLabelSelector }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "gatewaypodAntiAffinityPreferredDuringScheduling" . }} + {{- end }} +{{- end }} +{{- end }} + +{{- define "gatewaypodAntiAffinityRequiredDuringScheduling" }} + {{- range $index, $item := .podAntiAffinityLabelSelector }} + - labelSelector: + matchExpressions: + - key: {{ $item.key }} + operator: {{ $item.operator }} + {{- if $item.values }} + values: + {{- $vals := split "," $item.values }} + {{- range $i, $v := $vals }} + - {{ $v | quote }} + {{- end }} + {{- end }} + topologyKey: {{ $item.topologyKey }} + {{- end }} +{{- end }} + +{{- define "gatewaypodAntiAffinityPreferredDuringScheduling" }} + {{- range $index, $item := .podAntiAffinityTermLabelSelector }} + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: {{ $item.key }} + operator: {{ $item.operator }} + {{- if $item.values }} + values: + {{- $vals := split "," $item.values }} + {{- range $i, $v := $vals }} + - {{ $v | quote }} + {{- end }} + {{- end }} + topologyKey: {{ $item.topologyKey }} + weight: 100 + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_helpers.tpl new file mode 100755 index 000000000..bfc8bc400 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "gateway.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "gateway.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "gateway.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml new file mode 100755 index 000000000..2455ac345 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/autoscale.yaml @@ -0,0 +1,31 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if and $spec.enabled $spec.autoscaleEnabled $spec.autoscaleMin $spec.autoscaleMax }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ $key }} + namespace: {{ $spec.namespace | default $.Release.Namespace }} + labels: + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} +spec: + maxReplicas: {{ $spec.autoscaleMax }} + minReplicas: {{ $spec.autoscaleMin }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ $key }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ $spec.cpu.targetAverageUtilization }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/deployment.yaml new file mode 100755 index 000000000..4dcdef778 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/deployment.yaml @@ -0,0 +1,330 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if $spec.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $key }} + namespace: {{ $spec.namespace | default $.Release.Namespace }} + labels: + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} +spec: +{{- if not $spec.autoscaleEnabled }} +{{- if $spec.replicaCount }} + replicas: {{ $spec.replicaCount }} +{{- else }} + replicas: 1 +{{- end }} +{{- end }} + selector: + matchLabels: + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + strategy: + rollingUpdate: + maxSurge: {{ $spec.rollingMaxSurge }} + maxUnavailable: {{ $spec.rollingMaxUnavailable }} + template: + metadata: + labels: + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + annotations: + sidecar.istio.io/inject: "false" +{{- if $spec.podAnnotations }} +{{ toYaml $spec.podAnnotations | indent 8 }} +{{ end }} + spec: + serviceAccountName: {{ $key }}-service-account +{{- if $.Values.global.priorityClassName }} + priorityClassName: "{{ $.Values.global.priorityClassName }}" +{{- end }} +{{- if $.Values.global.proxy.enableCoreDump }} + initContainers: + - name: enable-core-dump + image: {{ $.Values.global.proxy.enableCoreDumpImage }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + command: + - /bin/sh + args: + - -c + - sysctl -w kernel.core_pattern=/var/lib/istio/core.proxy && ulimit -c unlimited + securityContext: + privileged: true +{{- end }} + containers: +{{- if $spec.sds }} +{{- if $spec.sds.enabled }} + - name: ingress-sds +{{- if contains "/" $spec.sds.image }} + image: "{{ $spec.sds.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $spec.sds.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + resources: +{{- if $spec.sds.resources }} +{{ toYaml $spec.sds.resources | indent 12 }} +{{- else }} +{{ toYaml $.Values.global.defaultResources | indent 12 }} +{{- end }} + env: + - name: "ENABLE_WORKLOAD_SDS" + value: "false" + - name: "ENABLE_INGRESS_GATEWAY_SDS" + value: "true" + - name: "INGRESS_GATEWAY_NAMESPACE" + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: ingressgatewaysdsudspath + mountPath: /var/run/ingress_gateway +{{- end }} +{{- end }} + - name: istio-proxy +{{- if contains "/" $.Values.global.proxy.image }} + image: "{{ $.Values.global.proxy.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $.Values.global.proxy.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + ports: + {{- range $key, $val := $spec.ports }} + - containerPort: {{ $val.port }} + {{- end }} + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - router + - --domain + - $(POD_NAMESPACE).svc.{{ $.Values.global.proxy.clusterDomain }} + {{- if $.Values.global.proxy.logLevel }} + - --proxyLogLevel={{ $.Values.global.proxy.logLevel }} + {{- end}} + {{- if $.Values.global.proxy.componentLogLevel }} + - --proxyComponentLogLevel={{ $.Values.global.proxy.componentLogLevel }} + {{- end}} + {{- if $.Values.global.logging.level }} + - --log_output_level={{ $.Values.global.logging.level }} + {{- end}} + - --drainDuration + - '45s' #drainDuration + - --parentShutdownDuration + - '1m0s' #parentShutdownDuration + - --connectTimeout + - '10s' #connectTimeout + - --serviceCluster + - {{ $key }} + - --zipkinAddress + {{- if $.Values.global.tracer.zipkin.address }} + - {{ $.Values.global.tracer.zipkin.address }} + {{- else if $.Values.global.istioNamespace }} + - zipkin.{{ $.Values.global.istioNamespace }}:9411 + {{- else }} + - zipkin:9411 + {{- end }} + {{- if $.Values.global.proxy.envoyStatsd.enabled }} + - --statsdUdpAddress + - {{ $.Values.global.proxy.envoyStatsd.host }}:{{ $.Values.global.proxy.envoyStatsd.port }} + {{- end }} + {{- if $.Values.global.proxy.envoyMetricsService.enabled }} + - --envoyMetricsServiceAddress + - {{ $.Values.global.proxy.envoyMetricsService.host }}:{{ $.Values.global.proxy.envoyMetricsService.port }} + {{- end }} + {{- if $.Values.global.proxy.envoyAccessLogService.enabled }} + - --envoyAccessLogService + {{- with $.Values.global.proxy.envoyAccessLogService }} + - '{"address":"{{ .host }}:{{.port }}"{{ if .tlsSettings }},"tlsSettings":{{ .tlsSettings | toJson }}{{- end }}{{ if .tcpKeepalive }},"tcpKeepalive":{{ .tcpKeepalive | toJson }}{{- end }}}' + {{- end }} + {{- end }} + - --proxyAdminPort + - "15000" + - --statusPort + - "15020" + {{- if $.Values.global.controlPlaneSecurityEnabled }} + - --controlPlaneAuthPolicy + - MUTUAL_TLS + - --discoveryAddress + {{- if $.Values.global.istioNamespace }} + - istio-pilot.{{ $.Values.global.istioNamespace }}:15011 + {{- else }} + - istio-pilot:15011 + {{- end }} + {{- else }} + - --controlPlaneAuthPolicy + - NONE + - --discoveryAddress + {{- if $.Values.global.istioNamespace }} + - istio-pilot.{{ $.Values.global.istioNamespace }}:15010 + {{- else }} + - istio-pilot:15010 + {{- end }} + {{- if $spec.applicationPorts }} + - --applicationPorts + - "{{ $spec.applicationPorts }}" + {{- end }} + {{- end }} + {{- if $.Values.global.trustDomain }} + - --trust-domain={{ $.Values.global.trustDomain }} + {{- end }} + readinessProbe: + failureThreshold: 30 + httpGet: + path: /healthz/ready + port: 15020 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 2 + successThreshold: 1 + timeoutSeconds: 1 + resources: +{{- if $spec.resources }} +{{ toYaml $spec.resources | indent 12 }} +{{- else }} +{{ toYaml $.Values.global.defaultResources | indent 12 }} +{{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: HOST_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: ISTIO_META_POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: ISTIO_META_CONFIG_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SDS_ENABLED + value: "{{ $.Values.global.sds.enabled }}" + - name: ISTIO_META_WORKLOAD_NAME + value: {{ $key }} + - name: ISTIO_META_OWNER + value: kubernetes://api/apps/v1/namespaces/{{ $spec.namespace | default $.Release.Namespace }}/deployments/{{ $key }} + {{- if $spec.sds }} + {{- if $spec.sds.enabled }} + - name: ISTIO_META_USER_SDS + value: "true" + {{- end }} + {{- end }} + {{- if $spec.env }} + {{- range $key, $val := $spec.env }} + - name: {{ $key }} + value: {{ $val }} + {{- end }} + {{- end }} + volumeMounts: + {{- if $.Values.global.sds.enabled }} + - name: sdsudspath + mountPath: /var/run/sds + readOnly: true + - name: istio-token + mountPath: /var/run/secrets/tokens + {{- end }} + {{- if $spec.sds }} + {{- if $spec.sds.enabled }} + - name: ingressgatewaysdsudspath + mountPath: /var/run/ingress_gateway + {{- end }} + {{- end }} + - name: istio-certs + mountPath: /etc/certs + readOnly: true + {{- range $spec.secretVolumes }} + - name: {{ .name }} + mountPath: {{ .mountPath | quote }} + readOnly: true + {{- end }} +{{- if $spec.additionalContainers }} +{{ toYaml $spec.additionalContainers | indent 8 }} +{{- end }} + volumes: + {{- if $spec.sds }} + {{- if $spec.sds.enabled }} + - name: ingressgatewaysdsudspath + emptyDir: {} + {{- end }} + {{- end }} + {{- if $.Values.global.sds.enabled }} + - name: sdsudspath + hostPath: + path: /var/run/sds + - name: istio-token + projected: + sources: + - serviceAccountToken: + path: istio-token + expirationSeconds: 43200 + audience: {{ $.Values.global.sds.token.aud }} + {{- end }} + - name: istio-certs + secret: + secretName: istio.{{ $key }}-service-account + optional: true + {{- range $spec.secretVolumes }} + - name: {{ .name }} + secret: + secretName: {{ .secretName | quote }} + optional: true + {{- end }} + {{- range $spec.configVolumes }} + - name: {{ .name }} + configMap: + name: {{ .configMapName | quote }} + optional: true + {{- end }} + affinity: + {{- include "gatewaynodeaffinity" (dict "root" $ "nodeSelector" $spec.nodeSelector) | indent 6 }} + {{- include "gatewaypodAntiAffinity" (dict "podAntiAffinityLabelSelector" $spec.podAntiAffinityLabelSelector "podAntiAffinityTermLabelSelector" $spec.podAntiAffinityTermLabelSelector) | indent 6 }} + {{- if $spec.tolerations }} + tolerations: +{{ toYaml $spec.tolerations | indent 6 }} + {{- else if $.Values.global.defaultTolerations }} + tolerations: +{{ toYaml $.Values.global.defaultTolerations | indent 6 }} + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..36a2d5a9c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/poddisruptionbudget.yaml @@ -0,0 +1,31 @@ +{{- range $key, $spec := .Values }} +{{- if and (ne $key "enabled") }} +{{- if $spec.enabled }} +{{- if $.Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ $key }} + namespace: {{ $spec.namespace | default $.Release.Namespace }} + labels: + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} +spec: +{{- if $.Values.global.defaultPodDisruptionBudget.enabled }} +{{ include "podDisruptionBudget.spec" $.Values.global.defaultPodDisruptionBudget }} +{{- end }} + selector: + matchLabels: + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/preconfigured.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/preconfigured.yaml new file mode 100755 index 000000000..8d3dee930 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/preconfigured.yaml @@ -0,0 +1,239 @@ +{{- if .Values.global.k8sIngress.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: istio-autogenerated-k8s-ingress + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + istio: {{ .Values.global.k8sIngress.gatewayName }} + servers: + - port: + number: 80 + protocol: HTTP2 + name: http + hosts: + - "*" +{{ if .Values.global.k8sIngress.enableHttps }} + - port: + number: 443 + protocol: HTTPS + name: https-default + tls: + mode: SIMPLE + serverCertificate: /etc/istio/ingress-certs/tls.crt + privateKey: /etc/istio/ingress-certs/tls.key + hosts: + - "*" +{{ end }} +--- +{{ end }} + +{{- if .Values.global.meshExpansion.enabled }} +{{- if .Values.global.meshExpansion.useILB }} +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: meshexpansion-ilb-gateway + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + istio: ilbgateway + servers: + - port: + number: 15011 + protocol: TCP + name: tcp-pilot + hosts: + - "*" + - port: + number: 8060 + protocol: TCP + name: tcp-citadel + hosts: + - "*" + - port: + number: 15004 + name: tls-mixer + protocol: TLS + tls: + mode: AUTO_PASSTHROUGH + hosts: + - "*" +--- +{{- else }} +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: meshexpansion-gateway + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + {{- range $key, $spec := .Values }} + {{- if eq $key "istio-ingressgateway" }} + {{- if $spec.enabled }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + servers: + - port: + number: 15011 + protocol: TCP + name: tcp-pilot + hosts: + - "*" + - port: + number: 8060 + protocol: TCP + name: tcp-citadel + hosts: + - "*" + - port: + number: 15004 + name: tls-mixer + protocol: TLS + tls: + mode: AUTO_PASSTHROUGH + hosts: + - "*" +--- +{{- end }} +{{- end }} + +{{- if .Values.global.multiCluster.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: istio-multicluster-egressgateway + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + {{- range $key, $spec := .Values }} + {{- if eq $key "istio-egressgateway" }} + {{- if $spec.enabled }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + servers: + - hosts: + - "*.global" + port: + name: tls + number: 15443 + protocol: TLS + tls: + mode: AUTO_PASSTHROUGH +--- +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: istio-multicluster-ingressgateway + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + {{- range $key, $spec := .Values }} + {{- if eq $key "istio-ingressgateway" }} + {{- if $spec.enabled }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + servers: + - hosts: + - "*.global" + port: + name: tls + number: 15443 + protocol: TLS + tls: + mode: AUTO_PASSTHROUGH +--- +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: istio-multicluster-ingressgateway + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + workloadLabels: + {{- range $key, $spec := .Values }} + {{- if eq $key "istio-ingressgateway" }} + {{- if $spec.enabled }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + filters: + - listenerMatch: + portNumber: 15443 + listenerType: GATEWAY + insertPosition: + index: AFTER + relativeTo: envoy.filters.network.sni_cluster + filterName: envoy.filters.network.tcp_cluster_rewrite + filterType: NETWORK + filterConfig: + cluster_pattern: "\\.global$" + cluster_replacement: ".svc.{{ .Values.global.proxy.clusterDomain }}" +--- +## To ensure all traffic to *.global is using mTLS +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: istio-multicluster-destinationrule + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "gateway.name" . }} + chart: {{ template "gateway.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: "*.global" + {{- if .Values.global.defaultConfigVisibilitySettings }} + exportTo: + - '*' + {{- end }} + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +--- +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/role.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/role.yaml new file mode 100755 index 000000000..37bdf3ef8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/role.yaml @@ -0,0 +1,18 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if $spec.enabled }} +{{- if ($spec.sds) and (eq $spec.sds.enabled true) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $key }}-sds + namespace: {{ $spec.namespace | default $.Release.Namespace }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list"] +--- +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/rolebindings.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/rolebindings.yaml new file mode 100755 index 000000000..cd3245b01 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/rolebindings.yaml @@ -0,0 +1,21 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if $spec.enabled }} +{{- if ($spec.sds) and (eq $spec.sds.enabled true) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $key }}-sds + namespace: {{ $spec.namespace | default $.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $key }}-sds +subjects: +- kind: ServiceAccount + name: {{ $key }}-service-account +--- +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/service.yaml new file mode 100755 index 000000000..9474f0476 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/service.yaml @@ -0,0 +1,59 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if $spec.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $key }} + namespace: {{ $spec.namespace | default $.Release.Namespace }} + annotations: + {{- range $key, $val := $spec.serviceAnnotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + labels: + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} +spec: +{{- if $spec.loadBalancerIP }} + loadBalancerIP: "{{ $spec.loadBalancerIP }}" +{{- end }} +{{- if $spec.loadBalancerSourceRanges }} + loadBalancerSourceRanges: +{{ toYaml $spec.loadBalancerSourceRanges | indent 4 }} +{{- end }} +{{- if $spec.externalTrafficPolicy }} + externalTrafficPolicy: {{$spec.externalTrafficPolicy }} +{{- end }} +{{- if $spec.externalIPs }} + externalIPs: +{{ toYaml $spec.externalIPs | indent 4 }} +{{- end }} + type: {{ .type }} + selector: + release: {{ $.Release.Name }} + {{- range $key, $val := $spec.labels }} + {{ $key }}: {{ $val }} + {{- end }} + ports: + {{- range $key, $val := $spec.ports }} + - + {{- range $pkey, $pval := $val }} + {{ $pkey}}: {{ $pval }} + {{- end }} + {{- end }} + {{- if $.Values.global.meshExpansion.enabled }} + {{- range $key, $val := $spec.meshExpansionPorts }} + - + {{- range $pkey, $pval := $val }} + {{ $pkey}}: {{ $pval }} + {{- end }} + {{- end }} + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/serviceaccount.yaml new file mode 100755 index 000000000..d4f6938c1 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/templates/serviceaccount.yaml @@ -0,0 +1,24 @@ +{{- range $key, $spec := .Values }} +{{- if ne $key "enabled" }} +{{- if $spec.enabled }} +apiVersion: v1 +kind: ServiceAccount +{{- if $.Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range $.Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: {{ $key }}-service-account + namespace: {{ $spec.namespace | default $.Release.Namespace }} + labels: + app: {{ $spec.labels.app }} + chart: {{ template "gateway.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} +--- +{{- end }} +{{- end }} +{{- end }} + diff --git a/kubernetes/helm_charts/istio-system/istio/charts/gateways/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/gateways/values.yaml new file mode 100755 index 000000000..2dc682af2 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/gateways/values.yaml @@ -0,0 +1,287 @@ +# +# Gateways Configuration +# By default (if enabled) a pair of Ingress and Egress Gateways will be created for the mesh. +# You can add more gateways in addition to the defaults but make sure those are uniquely named +# and that NodePorts are not conflicting. +# Disable specifc gateway by setting the `enabled` to false. +# +enabled: true + +istio-ingressgateway: + enabled: true + # + # Secret Discovery Service (SDS) configuration for ingress gateway. + # + sds: + # If true, ingress gateway fetches credentials from SDS server to handle TLS connections. + enabled: false + # SDS server that watches kubernetes secrets and provisions credentials to ingress gateway. + # This server runs in the same pod as ingress gateway. + image: node-agent-k8s + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 1024Mi + + labels: + app: istio-ingressgateway + istio: ingressgateway + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + # specify replicaCount when autoscaleEnabled: false + # replicaCount: 1 + rollingMaxSurge: 100% + rollingMaxUnavailable: 25% + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 1024Mi + cpu: + targetAverageUtilization: 80 + loadBalancerIP: "" + loadBalancerSourceRanges: [] + externalIPs: [] + serviceAnnotations: {} + podAnnotations: {} + type: LoadBalancer #change to NodePort, ClusterIP or LoadBalancer if need be + #externalTrafficPolicy: Local #change to Local to preserve source IP or Cluster for default behaviour or leave commented out + ports: + ## You can add custom gateway ports + # Note that AWS ELB will by default perform health checks on the first port + # on this list. Setting this to the health check port will ensure that health + # checks always work. https://github.com/istio/istio/issues/12503 + - port: 15020 + targetPort: 15020 + name: status-port + - port: 80 + targetPort: 80 + name: http2 + nodePort: 31380 + - port: 443 + name: https + nodePort: 31390 + # Example of a port to add. Remove if not needed + - port: 31400 + name: tcp + nodePort: 31400 + ### PORTS FOR UI/metrics ##### + ## Disable if not needed + - port: 15029 + targetPort: 15029 + name: https-kiali + - port: 15030 + targetPort: 15030 + name: https-prometheus + - port: 15031 + targetPort: 15031 + name: https-grafana + - port: 15032 + targetPort: 15032 + name: https-tracing + # This is the port where sni routing happens + - port: 15443 + targetPort: 15443 + name: tls + #### MESH EXPANSION PORTS ######## + # Pilot and Citadel MTLS ports are enabled in gateway - but will only redirect + # to pilot/citadel if global.meshExpansion settings are enabled. + # Delete these ports if mesh expansion is not enabled, to avoid + # exposing unnecessary ports on the web. + # You can remove these ports if you are not using mesh expansion + meshExpansionPorts: + - port: 15011 + targetPort: 15011 + name: tcp-pilot-grpc-tls + - port: 15004 + targetPort: 15004 + name: tcp-mixer-grpc-tls + - port: 8060 + targetPort: 8060 + name: tcp-citadel-grpc-tls + - port: 853 + targetPort: 853 + name: tcp-dns-tls + ####### end MESH EXPANSION PORTS ###### + ############## + secretVolumes: + - name: ingressgateway-certs + secretName: istio-ingressgateway-certs + mountPath: /etc/istio/ingressgateway-certs + - name: ingressgateway-ca-certs + secretName: istio-ingressgateway-ca-certs + mountPath: /etc/istio/ingressgateway-ca-certs + ### Advanced options ############ + + # Ports to explicitly check for readiness. If configured, the readiness check will expect a + # listener on these ports. A comma separated list is expected, such as "80,443". + # + # Warning: If you do not have a gateway configured for the ports provided, this check will always + # fail. This is intended for use cases where you always expect to have a listener on the port, + # such as 80 or 443 in typical setups. + applicationPorts: "" + + env: + # A gateway with this mode ensures that pilot generates an additional + # set of clusters for internal services but without Istio mTLS, to + # enable cross cluster routing. + ISTIO_META_ROUTER_MODE: "sni-dnat" + nodeSelector: {} + tolerations: [] + + # Specify the pod anti-affinity that allows you to constrain which nodes + # your pod is eligible to be scheduled based on labels on pods that are + # already running on the node rather than based on labels on nodes. + # There are currently two types of anti-affinity: + # "requiredDuringSchedulingIgnoredDuringExecution" + # "preferredDuringSchedulingIgnoredDuringExecution" + # which denote "hard" vs. "soft" requirements, you can define your values + # in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" + # correspondingly. + # For example: + # podAntiAffinityLabelSelector: + # - key: security + # operator: In + # values: S1,S2 + # topologyKey: "kubernetes.io/hostname" + # This pod anti-affinity rule says that the pod requires not to be scheduled + # onto a node if that node is already running a pod with label having key + # "security" and value "S1". + podAntiAffinityLabelSelector: [] + podAntiAffinityTermLabelSelector: [] + +istio-egressgateway: + enabled: false + labels: + app: istio-egressgateway + istio: egressgateway + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + # specify replicaCount when autoscaleEnabled: false + # replicaCount: 1 + rollingMaxSurge: 100% + rollingMaxUnavailable: 25% + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 1024Mi + cpu: + targetAverageUtilization: 80 + serviceAnnotations: {} + podAnnotations: {} + type: ClusterIP #change to NodePort or LoadBalancer if need be + ports: + - port: 80 + name: http2 + - port: 443 + name: https + # This is the port where sni routing happens + - port: 15443 + targetPort: 15443 + name: tls + secretVolumes: + - name: egressgateway-certs + secretName: istio-egressgateway-certs + mountPath: /etc/istio/egressgateway-certs + - name: egressgateway-ca-certs + secretName: istio-egressgateway-ca-certs + mountPath: /etc/istio/egressgateway-ca-certs + #### Advanced options ######## + env: + # Set this to "external" if and only if you want the egress gateway to + # act as a transparent SNI gateway that routes mTLS/TLS traffic to + # external services defined using service entries, where the service + # entry has resolution set to DNS, has one or more endpoints with + # network field set to "external". By default its set to "" so that + # the egress gateway sees the same set of endpoints as the sidecars + # preserving backward compatibility + # ISTIO_META_REQUESTED_NETWORK_VIEW: "" + # A gateway with this mode ensures that pilot generates an additional + # set of clusters for internal services but without Istio mTLS, to + # enable cross cluster routing. + ISTIO_META_ROUTER_MODE: "sni-dnat" + nodeSelector: {} + tolerations: [] + + # Specify the pod anti-affinity that allows you to constrain which nodes + # your pod is eligible to be scheduled based on labels on pods that are + # already running on the node rather than based on labels on nodes. + # There are currently two types of anti-affinity: + # "requiredDuringSchedulingIgnoredDuringExecution" + # "preferredDuringSchedulingIgnoredDuringExecution" + # which denote "hard" vs. "soft" requirements, you can define your values + # in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" + # correspondingly. + # For example: + # podAntiAffinityLabelSelector: + # - key: security + # operator: In + # values: S1,S2 + # topologyKey: "kubernetes.io/hostname" + # This pod anti-affinity rule says that the pod requires not to be scheduled + # onto a node if that node is already running a pod with label having key + # "security" and value "S1". + podAntiAffinityLabelSelector: [] + podAntiAffinityTermLabelSelector: [] + +# Mesh ILB gateway creates a gateway of type InternalLoadBalancer, +# for mesh expansion. It exposes the mtls ports for Pilot,CA as well +# as non-mtls ports to support upgrades and gradual transition. +istio-ilbgateway: + enabled: false + labels: + app: istio-ilbgateway + istio: ilbgateway + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + # specify replicaCount when autoscaleEnabled: false + # replicaCount: 1 + rollingMaxSurge: 100% + rollingMaxUnavailable: 25% + cpu: + targetAverageUtilization: 80 + resources: + requests: + cpu: 800m + memory: 512Mi + #limits: + # cpu: 1800m + # memory: 256Mi + loadBalancerIP: "" + serviceAnnotations: + cloud.google.com/load-balancer-type: "internal" + podAnnotations: {} + type: LoadBalancer + ports: + ## You can add custom gateway ports - google ILB default quota is 5 ports, + - port: 15011 + name: grpc-pilot-mtls + # Insecure port - only for migration from 0.8. Will be removed in 1.1 + - port: 15010 + name: grpc-pilot + - port: 8060 + targetPort: 8060 + name: tcp-citadel-grpc-tls + # Port 5353 is forwarded to kube-dns + - port: 5353 + name: tcp-dns + secretVolumes: + - name: ilbgateway-certs + secretName: istio-ilbgateway-certs + mountPath: /etc/istio/ilbgateway-certs + - name: ilbgateway-ca-certs + secretName: istio-ilbgateway-ca-certs + mountPath: /etc/istio/ilbgateway-ca-certs + nodeSelector: {} + tolerations: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/Chart.yaml new file mode 100755 index 000000000..ca0b464e5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: A Helm chart for Kubernetes +name: grafana +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/citadel-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/citadel-dashboard.json new file mode 100755 index 000000000..ffe4551a9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/citadel-dashboard.json @@ -0,0 +1,1089 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Performance", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "CPU usage across Citadel instances.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"citadel\", pod_name=~\"istio-citadel-.*\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Citadel CPU usage rate", + "refId": "A" + }, + { + "expr": "irate(process_cpu_seconds_total{job=\"citadel\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Citadel CPU usage irate", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Citadel process memory statistics.", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Virtual Memory", + "refId": "A" + }, + { + "expr": "process_resident_memory_bytes{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Resident Memory", + "refId": "B" + }, + { + "expr": "go_memstats_heap_sys_bytes{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Memory Total", + "refId": "C" + }, + { + "expr": "go_memstats_alloc_bytes{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Memory Allocated", + "refId": "E" + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Heap Inuse", + "refId": "F" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Goroutines", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 28, + "panels": [], + "title": "General", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Total number of CSR requests made to Citadel.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "citadel_server_csr_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CSR Request Count", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CSR Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of certificates issuances that have succeeded.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "citadel_server_success_cert_issuance_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Certificates Issued", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Certificates Issued", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 23, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of errors occurred when creating the CSR.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 20, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "citadel_secret_controller_csr_err_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CSR Creation Error Count", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CSR Creation Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "citadel_server_csr_parsing_err_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "CSR Parse Error Count", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CSR Parse Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of authentication failures.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "citadel_server_authentication_failure_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Authentication Failure Count", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Authentication Failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 4, + "panels": [], + "title": "Secret Controller", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of certificates created due to service account creation.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "citadel_secret_controller_svc_acc_created_cert_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SA Secrets Created", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Account Secrets Created (due to SA creation)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Certs Created", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of certificates deleted due to service account deletion.", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "citadel_secret_controller_svc_acc_deleted_cert_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SA Secrets Deleted", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Account Secrets Deleted (due to SA deletion)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Certs Created", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The number of certificates recreated due to secret deletion (service account still exists).", + "fill": 1, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "citadel_secret_controller_secret_deleted_cert_count{job=\"citadel\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "SA Secrets Recreated", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Service Account Secrets Recreated (due to errant deletion)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": "Certs Created", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Citadel Dashboard", + "uid": "OOyOqb4Wz", + "version": 1 +} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/galley-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/galley-dashboard.json new file mode 100755 index 000000000..b9b07daa9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/galley-dashboard.json @@ -0,0 +1,1819 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build{component=\"galley\"}) by (tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Galley Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 40, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Virtual Memory", + "refId": "A" + }, + { + "expr": "process_resident_memory_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Memory", + "refId": "B" + }, + { + "expr": "go_memstats_heap_sys_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap sys", + "refId": "C" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap alloc", + "refId": "D" + }, + { + "expr": "go_memstats_alloc_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc", + "refId": "F" + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap in-use", + "refId": "G" + }, + { + "expr": "go_memstats_stack_inuse_bytes{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Stack in-use", + "refId": "H" + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Total (kis)", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}[1m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}[1m])) by (container_name)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B" + }, + { + "expr": "irate(process_cpu_seconds_total{job=\"galley\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "galley (self-reported)", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Open FDs (galley)", + "refId": "A" + }, + { + "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ container_name }} ", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "goroutines_total", + "refId": "A" + }, + { + "expr": "istio_mcp_clients_total{component=\"galley\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "clients_total", + "refId": "B" + }, + { + "expr": "go_goroutines{job=\"galley\"}/sum(istio_mcp_clients_total{component=\"galley\"}) without (component)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "avg_goroutines_per_client", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 10, + "panels": [], + "title": "Runtime", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(galley_runtime_strategy_on_change_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Strategy Change Events", + "refId": "A" + }, + { + "expr": "sum(rate(galley_runtime_processor_events_processed_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Processed Events", + "refId": "B" + }, + { + "expr": "sum(rate(galley_runtime_processor_snapshots_published_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Snapshot Published", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Event Rates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Events/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(galley_runtime_strategy_timer_max_time_reached_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Max Time Reached", + "refId": "A" + }, + { + "expr": "sum(rate(galley_runtime_strategy_timer_quiesce_reached_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Quiesce Reached", + "refId": "B" + }, + { + "expr": "sum(rate(galley_runtime_strategy_timer_resets_total[1m])) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Timer Resets", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Timer Rates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Events/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 3, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.90, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P95", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Events Per Snapshot", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 21 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (typeURL) (galley_runtime_state_type_instances_total)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ typeURL }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "State Type Instances", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 34, + "panels": [], + "title": "Validation", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 28, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "galley_validation_cert_key_updates{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Key Updates", + "refId": "A" + }, + { + "expr": "galley_validation_cert_key_update_errors{job=\"galley\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Key Update Errors: {{ error }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Validation Webhook Certificate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 30, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(galley_validation_passed{job=\"galley\"}) by (group, version, resource)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Passed: {{ group }}/{{ version }}/{{resource}}", + "refId": "A" + }, + { + "expr": "sum(galley_validation_failed{job=\"galley\"}) by (group, version, resource, reason)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Failed: {{ group }}/{{ version }}/{{resource}} ({{ reason}})", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resource Validation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 28 + }, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(galley_validation_http_error{job=\"galley\"}) by (status)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ status }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Validation HTTP Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 12, + "panels": [], + "title": "Kubernetes Source", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 35 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(galley_source_kube_event_success_total[1m]) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Success", + "refId": "A" + }, + { + "expr": "rate(galley_source_kube_event_error_total[1m]) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Error", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Source Event Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Events/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 35 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(galley_source_kube_dynamic_converter_success_total[1m]) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{apiVersion=\"{{apiVersion}}\",group=\"{{group}}\",kind=\"{{kind}}\"}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Kubernetes Object Conversion Successes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Conversions/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 35 + }, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(galley_source_kube_dynamic_converter_failure_total[1m]) * 60", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Error", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Kubernetes Object Conversion Failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Failures/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 18, + "panels": [], + "title": "Mesh Configuration Protocol", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 42 + }, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_mcp_clients_total{component=\"galley\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Clients", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Connected Clients", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 42 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(collection)(irate(istio_mcp_request_acks_total{component=\"galley\"}[1m]) * 60)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request ACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "ACKs/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 42 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(istio_mcp_request_nacks_total{component=\"galley\"}[1m]) * 60", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request NACKs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "NACKs/min", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Galley Dashboard", + "uid": "TSEY6jLmk", + "version": 1 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-mesh-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-mesh-dashboard.json new file mode 100755 index 000000000..1662e1f1d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-mesh-dashboard.json @@ -0,0 +1,1225 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "content": "
\n
\n Istio\n
\n
\n Istio is an open platform that provides a uniform way to connect,\n manage, and \n secure microservices.\n
\n Need help? Join the Istio community.\n
\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "height": "50px", + "id": 13, + "links": [], + "mode": "html", + "style": { + "font-size": "18pt" + }, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\"}[1m])), 0.001)", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Global Request Volume", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Global Success Rate (non-5xx responses)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 22, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"4.*\"}[1m])) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "4xxs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 23, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"5.*\"}[1m])) ", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "5xxs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 113, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(galley_istio_networking_virtualservices) / count(up{job=\"galley\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Virtual Services", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 114, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(galley_istio_networking_destinationrules) / count(up{job=\"galley\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Destination Rules", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 115, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(galley_istio_networking_gateways) / count(up{job=\"galley\"})", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Gateways", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 116, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(galley_istio_authentication_meshpolicies) / count(up{job=\"galley\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "timeFrom": null, + "timeShift": null, + "title": "Authentication Mesh Policies", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 21, + "w": 24, + "x": 0, + "y": 9 + }, + "hideTimeOverride": false, + "id": 73, + "links": [], + "pageSize": null, + "repeat": null, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 4, + "desc": true + }, + "styles": [ + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Workload dashboard", + "linkUrl": "/dashboard/db/istio-workload-dashboard?var-namespace=$__cell_2&var-workload=$__cell_", + "pattern": "destination_workload", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "s" + }, + { + "alias": "Success Rate", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + ".95", + " 1.00" + ], + "type": "number", + "unit": "percentunit" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell dashboard", + "linkUrl": "/dashboard/db/istio-workload-dashboard?var-workload=$__cell_2&var-namespace=$__cell_3", + "pattern": "destination_workload_var", + "thresholds": [], + "type": "number", + "unit": "short" + }, + { + "alias": "Service", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell dashboard", + "linkUrl": "/dashboard/db/istio-service-dashboard?var-service=$__cell", + "pattern": "destination_service", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "destination_workload_namespace", + "thresholds": [], + "type": "hidden", + "unit": "short" + } + ], + "targets": [ + { + "expr": "label_join(sum(rate(istio_requests_total{reporter=\"destination\", response_code=\"200\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", + "refId": "A" + }, + { + "expr": "label_join((histogram_quantile(0.50, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.50, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", + "refId": "B" + }, + { + "expr": "label_join((histogram_quantile(0.90, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.90, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "D" + }, + { + "expr": "label_join((histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.99, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "E" + }, + { + "expr": "label_join((sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) by (destination_workload, destination_workload_namespace) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m])) by (destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "F" + } + ], + "timeFrom": null, + "title": "HTTP/GRPC Workloads", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "Prometheus", + "fontSize": "100%", + "gridPos": { + "h": 18, + "w": 24, + "x": 0, + "y": 30 + }, + "hideTimeOverride": false, + "id": 109, + "links": [], + "pageSize": null, + "repeatDirection": "v", + "scroll": true, + "showHeader": true, + "sort": { + "col": 2, + "desc": true + }, + "styles": [ + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "$__cell dashboard", + "linkUrl": "/dashboard/db/istio-tcp-workload-dashboard?var-namespace=$__cell_2&&var-workload=$__cell", + "pattern": "destination_workload", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Bytes Sent", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "Bytes Received", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Workload", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell dashboard", + "linkUrl": "/dashboard/db/istio-workload-dashboard?var-namespace=$__cell_3&var-workload=$__cell_2", + "pattern": "destination_workload_var", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "destination_workload_namespace", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Service", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "$__cell dashboard", + "linkUrl": "/dashboard/db/istio-service-dashboard?var-service=$__cell", + "pattern": "destination_service", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "label_join(sum(rate(istio_tcp_received_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}", + "refId": "C" + }, + { + "expr": "label_join(sum(rate(istio_tcp_sent_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}", + "refId": "A" + } + ], + "timeFrom": null, + "title": "TCP Workloads", + "transform": "table", + "type": "table" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 111, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build) by (component, tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ component }}: {{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Istio Components by Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Istio Mesh Dashboard", + "uid": "G8wLrJIZk", + "version": 5 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-performance-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-performance-dashboard.json new file mode 100755 index 000000000..3826fbae8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-performance-dashboard.json @@ -0,0 +1,1822 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": 9, + "links": [], + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 21, + "panels": [ + { + "content": "The charts on this dashboard are intended to show Istio main components cost in terms resources utilization under steady load.\n\n- **vCPU/1k rps:** shows vCPU utilization by the main Istio components normalized by 1000 requests/second. When idle or low traffic, this chart will be blank. The curve for istio-proxy refers to the services sidecars only.\n- **vCPU:** vCPU utilization by Istio components, not normalized.\n- **Memory:** memory footprint for the components. Telemetry and policy are normalized by 1k rps, and no data is shown when there is no traffic. For ingress and istio-proxy, the data is per instance.\n- **Bytes transferred/ sec:** shows the number of bytes flowing through each Istio component.\n\n\n", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 19, + "links": [], + "mode": "markdown", + "timeFrom": null, + "timeShift": null, + "title": "Performance Dashboard README", + "transparent": true, + "type": "text" + } + ], + "title": "Performance Dashboard Notes", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 6, + "panels": [], + "title": "vCPU Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 2 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "istio-telemetry", + "refId": "A" + }, + { + "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name=\"istio-proxy\"}[1m])) / (round(sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\", reporter=\"source\"}[1m])), 0.001)/1000))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "istio-ingressgateway", + "refId": "B" + }, + { + "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-proxy", + "refId": "C" + }, + { + "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-policy", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "vCPU / 1k rps", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-telemetry", + "refId": "A" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name=\"istio-proxy\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-ingressgateway", + "refId": "B" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-proxy", + "refId": "C" + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-policy", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "vCPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 13, + "panels": [], + "title": "Memory and Data Rates", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 902, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\"}) / (sum(irate(istio_requests_total[1m])) / 1000)) / (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-telemetry / 1k rps", + "refId": "A" + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\"}) / count(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name!=\"POD\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "per istio-ingressgateway", + "refId": "B" + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}) / count(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "per istio proxy", + "refId": "C" + }, + { + "expr": "(sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\"}) / (sum(irate(istio_requests_total[1m])) / 1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-policy / 1k rps", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_response_bytes_sum{destination_workload=\"istio-telemetry\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload=\"istio-telemetry\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-telemetry", + "refId": "A" + }, + { + "expr": "sum(irate(istio_response_bytes_sum{source_workload=\"istio-ingressgateway\", reporter=\"source\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-ingressgateway", + "refId": "B" + }, + { + "expr": "sum(irate(istio_response_bytes_sum{source_workload_namespace!=\"istio-system\", reporter=\"source\"}[1m])) + sum(irate(istio_response_bytes_sum{destination_workload_namespace!=\"istio-system\", reporter=\"destination\"}[1m])) + sum(irate(istio_request_bytes_sum{source_workload_namespace!=\"istio-system\", reporter=\"source\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload_namespace!=\"istio-system\", reporter=\"destination\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio-proxy", + "refId": "C" + }, + { + "expr": "sum(irate(istio_response_bytes_sum{destination_workload=\"istio-policy\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload=\"istio-policy\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "istio_policy", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes transferred / sec", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 17, + "panels": [], + "title": "Istio Component Versions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build) by (component, tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ component }}: {{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Istio Components by Version", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 71, + "panels": [], + "title": "Proxy Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 32 + }, + "id": 72, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 32 + }, + "id": 73, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "vCPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 32 + }, + "id": 702, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"istio-proxy\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ container_name }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1024, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 69, + "panels": [], + "title": "Pilot Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 40 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{job=\"pilot\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "Virtual Memory", + "refId": "I", + "step": 2 + }, + { + "expr": "process_resident_memory_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Memory", + "refId": "H", + "step": 2 + }, + { + "expr": "go_memstats_heap_sys_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap sys", + "refId": "A" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap alloc", + "refId": "D" + }, + { + "expr": "go_memstats_alloc_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc", + "refId": "F", + "step": 2 + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Heap in-use", + "refId": "E", + "step": 2 + }, + { + "expr": "go_memstats_stack_inuse_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Stack in-use", + "refId": "G", + "step": 2 + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "C", + "step": 2 + }, + { + "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 602, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m])) by (container_name)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B", + "step": 2 + }, + { + "expr": "irate(process_cpu_seconds_total{job=\"pilot\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "pilot (self-reported)", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "vCPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 40 + }, + "id": 74, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{job=\"pilot\"}", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open FDs (pilot)", + "refId": "A" + }, + { + "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ container_name }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1024, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 40 + }, + "id": 402, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of Goroutines", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 93, + "panels": [], + "title": "Mixer Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 94, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "Virtual Memory", + "refId": "I", + "step": 2 + }, + { + "expr": "process_resident_memory_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Memory", + "refId": "H", + "step": 2 + }, + { + "expr": "go_memstats_heap_sys_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap sys", + "refId": "A" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap alloc", + "refId": "D" + }, + { + "expr": "go_memstats_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc", + "refId": "F", + "step": 2 + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Heap in-use", + "refId": "E", + "step": 2 + }, + { + "expr": "go_memstats_stack_inuse_bytes{job=~\"istio-policy|istio-telemetry\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Stack in-use", + "refId": "G", + "step": 2 + }, + { + "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"})", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "C", + "step": 2 + }, + { + "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 48 + }, + "id": 95, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total (k8s)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}[1m])) by (container_name)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ container_name }} (k8s)", + "refId": "B", + "step": 2 + }, + { + "expr": "irate(process_cpu_seconds_total{job=~\"istio-policy|istio-telemetry\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "mixer (self-reported)", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "vCPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 48 + }, + "id": 96, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_open_fds{job=~\"istio-policy|istio-telemetry\"}", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open FDs (pilot)", + "refId": "A" + }, + { + "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ container_name }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1024, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 48 + }, + "id": 97, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"istio-telemetry\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of Goroutines", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Performance Dashboard", + "uid": "vu8e0VWZk", + "version": 22 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-service-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-service-dashboard.json new file mode 100755 index 000000000..f4d58a296 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-service-dashboard.json @@ -0,0 +1,2601 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1536442501501, + "links": [], + "panels": [ + { + "content": "
\nSERVICE: $service\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 89, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Client Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Client Success Rate (non-5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 87, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Client Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Received Bytes", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 97, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Server Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 98, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Server Success Rate (non-5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 99, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Server Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 100, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m])) ", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Sent Bytes", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "content": "
\nCLIENT WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 45, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"source\",source_workload=~\"$srcwl\",source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"source\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests by Source And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 80, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "
\nSERVICE WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 69, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 90, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"destination\",destination_workload=~\"$dstwl\",destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"destination\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests by Destination And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 91, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 94, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 95, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 96, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 93, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Service", + "multi": false, + "name": "service", + "options": [], + "query": "label_values(destination_service)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Client Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Client Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Service Workload Namespace", + "multi": true, + "name": "dstns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (destination_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Service Workload", + "multi": true, + "name": "dstwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Service Dashboard", + "uid": "LJ_uJAvmk", + "version": 1 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-workload-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-workload-dashboard.json new file mode 100755 index 000000000..62ad1b5cf --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/istio-workload-dashboard.json @@ -0,0 +1,2303 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.4" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "iteration": 1531345461465, + "links": [], + "panels": [ + { + "content": "
\nWORKLOAD: $workload.$namespace\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 89, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Incoming Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "decimals": null, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 14, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": "95, 99, 99.5", + "title": "Incoming Success Rate (non-5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 87, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "P50", + "refId": "A" + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P90", + "refId": "B" + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "P99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Request Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 84, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Server Traffic", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "Bps", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 85, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "TCP Client Traffic", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "content": "
\nINBOUND WORKLOADS\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 45, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests by Source And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 26, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Success Rate (non-5xx responses) By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 27, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Duration by Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Request Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 68, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Size By Source", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 80, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Received from Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 82, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Sent to Incoming TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "content": "
\nOUTBOUND SERVICES\n
", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 69, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 0, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 70, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }} (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} : {{ response_code }}", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Outgoing Requests by Destination And Response Code", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 71, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Outgoing Success Rate (non-5xx responses) By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": "1.01", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 72, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Outgoing Request Duration by Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 41 + }, + "id": 73, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Outgoing Request Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 74, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", + "refId": "D", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", + "refId": "B", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", + "refId": "C", + "step": 2 + }, + { + "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P50", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P90", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P95", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} P99", + "refId": "H", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Size By Destination", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Sent on Outgoing TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 78, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }} (🔐mTLS)", + "refId": "A", + "step": 2 + }, + { + "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Bytes Received from Outgoing TCP Connection", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "query_result(sum(istio_requests_total) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total) by (destination_workload_namespace))", + "refresh": 1, + "regex": "/.*_namespace=\"([^\"]*).*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Workload", + "multi": false, + "name": "workload", + "options": [], + "query": "query_result((sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_requests_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)) or (sum(istio_tcp_sent_bytes_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload Namespace", + "multi": true, + "name": "srcns", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace))", + "refresh": 1, + "regex": "/.*namespace=\"([^\"]*).*/", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Inbound Workload", + "multi": true, + "name": "srcwl", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", + "refresh": 1, + "regex": "/.*workload=\"([^\"]*).*/", + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Destination Service", + "multi": true, + "name": "dstsvc", + "options": [], + "query": "query_result( sum(istio_requests_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service))", + "refresh": 1, + "regex": "/.*destination_service=\"([^\"]*).*/", + "sort": 4, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Workload Dashboard", + "uid": "UbsSZTDik", + "version": 1 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/mixer-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/mixer-dashboard.json new file mode 100755 index 000000000..b074d13ba --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/mixer-dashboard.json @@ -0,0 +1,1808 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "limit": 100, + "name": "Annotations & Alerts", + "showIn": 0, + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1543881232533, + "links": [], + "panels": [ + { + "content": "

Deployed Versions

", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "height": "40", + "id": 62, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 64, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build{component=\"mixer\"}) by (tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Mixer Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "

Resource Usage

", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 8 + }, + "height": "40", + "id": 29, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(process_virtual_memory_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "Virtual Memory ({{ job }})", + "refId": "I" + }, + { + "expr": "sum(process_resident_memory_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Memory ({{ job }})", + "refId": "H" + }, + { + "expr": "sum(go_memstats_heap_sys_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap sys ({{ job }})", + "refId": "A" + }, + { + "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap alloc ({{ job }})", + "refId": "D" + }, + { + "expr": "sum(go_memstats_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc ({{ job }})", + "refId": "F" + }, + { + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Heap in-use ({{ job }})", + "refId": "E" + }, + { + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Stack in-use ({{ job }})", + "refId": "G" + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (service)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ service }} total (k8s)", + "refId": "C" + }, + { + "expr": "sum(label_replace(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (container_name, service)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ service }} - {{ container_name }} (k8s)", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 11 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}[1m])) by (pod_name), \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ service }} total (k8s)", + "refId": "A" + }, + { + "expr": "label_replace(sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}[1m])) by (container_name, pod_name), \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ service }} - {{ container_name }} (k8s)", + "refId": "B" + }, + { + "expr": "sum(irate(process_cpu_seconds_total{job=~\"istio-telemetry|istio-policy\"}[1m])) by (job)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ job }} (self-reported)", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 11 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(process_open_fds{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "Open FDs ({{ job }})", + "refId": "A" + }, + { + "expr": "sum(label_replace(container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (container_name, service)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ service }} - {{ container_name }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1024, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 11 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(go_goroutines{job=~\"istio-telemetry|istio-policy\"}) by (job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of Goroutines ({{ job }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "

Mixer Overview

", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 18 + }, + "height": "40px", + "id": 30, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 21 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_io_server_completed_rpcs[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "mixer (Total)", + "refId": "B" + }, + { + "expr": "sum(rate(grpc_io_server_completed_rpcs[1m])) by (grpc_server_method)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mixer ({{ grpc_server_method }})", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Incoming Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "{}", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ grpc_server_method }} 0.5", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ grpc_server_method }} 0.9", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ grpc_server_method }} 0.99", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Response Durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 21 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|Unimplemented|Internal|DataLoss\"}[1m])) by (grpc_method)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Mixer {{ grpc_method }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Server Error Rate (5xx responses)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 21 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(grpc_server_handled_total{grpc_code!=\"OK\",grpc_service=~\".*Mixer\"}[1m])) by (grpc_method)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Mixer {{ grpc_method }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Non-successes (4xxs)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "

Adapters and Config

", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 28, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(mixer_runtime_dispatches_total{adapter=~\"$adapter\"}[1m])) by (adapter)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ adapter }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Adapter Dispatch Count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ adapter }} - p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ adapter }} - p90 ", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ adapter }} - p99", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Adapter Dispatch Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(topk(1, max(mixer_config_rule_config_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Rules", + "refId": "A" + }, + { + "expr": "scalar(topk(1, max(mixer_config_rule_config_error_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Config Errors", + "refId": "B" + }, + { + "expr": "scalar(topk(1, max(mixer_config_rule_config_match_error_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Match Errors", + "refId": "C" + }, + { + "expr": "scalar(topk(1, max(mixer_config_unsatisfied_action_handler_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Unsatisfied Actions", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Rules", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(topk(1, max(mixer_config_instance_config_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Instances", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Instances in Latest Config", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 54, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(topk(1, max(mixer_config_handler_config_count) by (configID)))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Handlers", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Handlers in Latest Config", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "scalar(topk(1, max(mixer_config_attribute_count) by (configID)))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "Attributes", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Attributes in Latest Config", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "content": "

Individual Adapters

", + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 23, + "links": [], + "mode": "html", + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 46, + "panels": [], + "repeat": "adapter", + "title": "$adapter Adapter", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(irate(mixer_runtime_dispatches_total{adapter=~\"$adapter\"}[1m]),\"handler\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ handler }} (error: {{ error }})", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Dispatch Count By Handler", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "label_replace(histogram_quantile(0.5, sum(rate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p50 - {{ handler_short }} (error: {{ error }})", + "refId": "A" + }, + { + "expr": "label_replace(histogram_quantile(0.9, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p90 - {{ handler_short }} (error: {{ error }})", + "refId": "D" + }, + { + "expr": "label_replace(histogram_quantile(0.99, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "p99 - {{ handler_short }} (error: {{ error }})", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Dispatch Duration By Handler", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Adapter", + "multi": true, + "name": "adapter", + "options": [], + "query": "label_values(adapter)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Istio Mixer Dashboard", + "version": 4 +} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/pilot-dashboard.json b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/pilot-dashboard.json new file mode 100755 index 000000000..b8ef6d2d7 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/dashboards/pilot-dashboard.json @@ -0,0 +1,1591 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 1, + "id": 11, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 60, + "panels": [], + "title": "Deployed Versions", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 56, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(istio_build{component=\"pilot\"}) by (tag)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ tag }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pilot Versions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 62, + "panels": [], + "title": "Resource Usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "process_virtual_memory_bytes{job=\"pilot\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "legendFormat": "Virtual Memory", + "refId": "I", + "step": 2 + }, + { + "expr": "process_resident_memory_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Resident Memory", + "refId": "H", + "step": 2 + }, + { + "expr": "go_memstats_heap_sys_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap sys", + "refId": "A" + }, + { + "expr": "go_memstats_heap_alloc_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": true, + "intervalFactor": 2, + "legendFormat": "heap alloc", + "refId": "D" + }, + { + "expr": "go_memstats_alloc_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alloc", + "refId": "F", + "step": 2 + }, + { + "expr": "go_memstats_heap_inuse_bytes{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Heap in-use", + "refId": "E", + "step": 2 + }, + { + "expr": "go_memstats_stack_inuse_bytes{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Stack in-use", + "refId": "G", + "step": 2 + }, + { + "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"discovery\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Discovery (container)", + "refId": "B", + "step": 2 + }, + { + "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sidecar (container)", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"discovery\", pod_name=~\"istio-pilot-.*\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Discovery (container)", + "refId": "A" + }, + { + "expr": "irate(process_cpu_seconds_total{job=\"pilot\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Discovery (process)", + "refId": "C", + "step": 2 + }, + { + "expr": "sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Sidecar (container)", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"discovery\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Discovery", + "refId": "B", + "step": 2 + }, + { + "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Sidecar", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": null, + "format": "none", + "label": "", + "logBase": 1024, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of Goroutines", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 58, + "panels": [], + "title": "Pilot Push Information", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "description": "Shows the rate of pilot pushes", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 622, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "paceLength": 10, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(pilot_xds_pushes{type=\"cds\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Cluster", + "refId": "C" + }, + { + "expr": "sum(irate(pilot_xds_pushes{type=\"eds\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Endpoints", + "refId": "D" + }, + { + "expr": "sum(irate(pilot_xds_pushes{type=\"lds\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Listeners", + "refId": "A" + }, + { + "expr": "sum(irate(pilot_xds_pushes{type=\"rds\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Routes", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pilot Pushes", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "total" + ] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Captures a variety of pilot errors", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 67, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(pilot_xds_cds_reject{job=\"pilot\"}) or (absent(pilot_xds_cds_reject{job=\"pilot\"}) - 1)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Rejected CDS Configs", + "refId": "C" + }, + { + "expr": "sum(pilot_xds_eds_reject{job=\"pilot\"}) or (absent(pilot_xds_eds_reject{job=\"pilot\"}) - 1)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Rejected EDS Configs", + "refId": "D" + }, + { + "expr": "sum(pilot_xds_rds_reject{job=\"pilot\"}) or (absent(pilot_xds_rds_reject{job=\"pilot\"}) - 1)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Rejected RDS Configs", + "refId": "A" + }, + { + "expr": "sum(pilot_xds_lds_reject{job=\"pilot\"}) or (absent(pilot_xds_lds_reject{job=\"pilot\"}) - 1)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Rejected LDS Configs", + "refId": "B" + }, + { + "expr": "sum(rate(pilot_xds_write_timeout{job=\"pilot\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Write Timeouts", + "refId": "F" + }, + { + "expr": "sum(rate(pilot_total_xds_internal_errors{job=\"pilot\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Internal Errors", + "refId": "H" + }, + { + "expr": "sum(rate(pilot_total_xds_rejects{job=\"pilot\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Config Rejection Rate", + "refId": "E" + }, + { + "expr": "sum(rate(pilot_xds_push_context_errors{job=\"pilot\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Push Context Errors", + "refId": "K" + }, + { + "expr": "sum(rate(pilot_xds_pushes{type!~\"lds|cds|rds|eds\"}[1m])) by (type)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Push Errors ({{ type }})", + "refId": "L" + }, + { + "expr": "sum(rate(pilot_xds_push_errors{job=\"pilot\"}[1m])) by (type)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Push Errors ({{ type }})", + "refId": "I" + }, + { + "expr": "sum(rate(pilot_xds_push_timeout{job=\"pilot\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Push Timeouts", + "refId": "G" + }, + { + "expr": "sum(rate(pilot_xds_push_timeout_failures{job=\"pilot\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Push Timeouts Failures", + "refId": "J" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pilot Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Shows the total time it takes to push a config update to a proxy", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 624, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.5, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p50 ", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.9, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p90", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.999, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "p99.9", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Proxy Push Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 45, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pilot_conflict_inbound_listener{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Inbound Listeners", + "refId": "B" + }, + { + "expr": "pilot_conflict_outbound_listener_http_over_current_tcp{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Outbound Listeners (http over current tcp)", + "refId": "A" + }, + { + "expr": "pilot_conflict_outbound_listener_tcp_over_current_tcp{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Outbound Listeners (tcp over current tcp)", + "refId": "C" + }, + { + "expr": "pilot_conflict_outbound_listener_tcp_over_current_http{job=\"pilot\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Outbound Listeners (tcp over current http)", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Conflicts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 47, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "pilot_virt_services{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Virtual Services", + "refId": "A" + }, + { + "expr": "pilot_services{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Services", + "refId": "B" + }, + { + "expr": "pilot_xds{job=\"pilot\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Connected Endpoints", + "refId": "E" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "ADS Monitoring", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "Prometheus", + "description": "Clusters in this table do not have any endpoints known to pilot. This could be from referencing subsets that do not have any instances, or pods marked as NotReady", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 51, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": null, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "Clusters", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "sum(pilot_xds_eds_instances{job=\"pilot\", cluster=~\".+\\\\|.+\"}) by (cluster) < 1", + "format": "time_series", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{cluster}}", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Clusters with no known endpoints", + "transform": "timeseries_aggregations", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 64, + "panels": [], + "title": "Envoy Information", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Shows details about Envoy proxies in the mesh", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 32 + }, + "id": 40, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(envoy_cluster_upstream_cx_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Connections", + "refId": "C" + }, + { + "expr": "sum(irate(envoy_cluster_upstream_cx_connect_fail{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Connection Failures", + "refId": "A" + }, + { + "expr": "sum(increase(envoy_server_hot_restart_epoch[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Envoy Restarts", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Envoy Details", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 32 + }, + "id": 41, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(envoy_cluster_upstream_cx_active{cluster_name=\"xds-grpc\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "XDS Active Connections", + "refId": "C", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "XDS Active Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "description": "Shows the size of XDS requests and responses", + "fill": 1, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 32 + }, + "id": 42, + "legend": { + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Response Bytes Max", + "refId": "D" + }, + { + "expr": "quantile(0.5, rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "XDS Response Bytes Average", + "refId": "B" + }, + { + "expr": "max(rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "XDS Request Bytes Max", + "refId": "A" + }, + { + "expr": "quantile(.5, rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "XDS Request Bytes Average", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "XDS Requests Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Istio Pilot Dashboard", + "uid": "3--MLVZZk", + "version": 11 +} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/_helpers.tpl new file mode 100755 index 000000000..9d4c59205 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "grafana.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "grafana.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "grafana.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-custom-resources.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-custom-resources.yaml new file mode 100755 index 000000000..b89bc0765 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-custom-resources.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-grafana-custom-resources + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: grafana +data: + custom-resources.yaml: |- + {{- include "grafana-default.yaml.tpl" . | indent 4}} + run.sh: |- + {{- include "install-custom-resources.sh.tpl" . | indent 4}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-dashboards.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-dashboards.yaml new file mode 100755 index 000000000..dd1ab0d75 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap-dashboards.yaml @@ -0,0 +1,18 @@ +{{- $files := .Files }} +{{- range $path, $bytes := .Files.Glob "dashboards/*.json" }} +{{- $filename := trimSuffix (ext $path) (base $path) }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-grafana-configuration-dashboards-{{ $filename }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ template "grafana.name" $ }} + chart: {{ template "grafana.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + istio: grafana +data: + {{ base $path }}: '{{ $files.Get $path }}' +--- +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap.yaml new file mode 100755 index 000000000..c86efe1f4 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/configmap.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: grafana +data: +{{- if .Values.datasources }} + {{- range $key, $value := .Values.datasources }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} + +{{- if .Values.dashboardProviders }} + {{- range $key, $value := .Values.dashboardProviders }} + {{ $key }}: | +{{ toYaml $value | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/create-custom-resources-job.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/create-custom-resources-job.yaml new file mode 100755 index 000000000..8f179d5cc --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/create-custom-resources-job.yaml @@ -0,0 +1,101 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-grafana-post-install-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-grafana-post-install-{{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["authentication.istio.io"] # needed to create default authn policy + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-grafana-post-install-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-grafana-post-install-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-grafana-post-install-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: istio-grafana-post-install-{{ .Values.global.tag | printf "%v" | trunc 32 }} + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-delete-policy": hook-succeeded + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + template: + metadata: + name: istio-grafana-post-install + labels: + app: istio-grafana + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + serviceAccountName: istio-grafana-post-install-account + containers: + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + command: [ "/bin/bash", "/tmp/grafana/run.sh", "/tmp/grafana/custom-resources.yaml" ] + volumeMounts: + - mountPath: "/tmp/grafana" + name: tmp-configmap-grafana + volumes: + - name: tmp-configmap-grafana + configMap: + name: istio-grafana-custom-resources + restartPolicy: OnFailure + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/deployment.yaml new file mode 100755 index 000000000..ba3037e04 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/deployment.yaml @@ -0,0 +1,138 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + prometheus.io/scrape: "true" + spec: + securityContext: + runAsUser: 472 + fsGroup: 472 +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: 3000 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + env: + - name: GRAFANA_PORT + value: "3000" +{{- if .Values.security.enabled }} + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: {{ .Values.security.secretName }} + key: {{ .Values.security.usernameKey }} + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.security.secretName }} + key: {{ .Values.security.passphraseKey }} + - name: GF_AUTH_BASIC_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "false" + - name: GF_AUTH_DISABLE_LOGIN_FORM + value: "false" +{{- else }} + - name: GF_AUTH_BASIC_ENABLED + value: "false" + - name: GF_AUTH_ANONYMOUS_ENABLED + value: "true" + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Admin +{{- end }} + - name: GF_PATHS_DATA + value: /data/grafana + {{- range $key, $value := $.Values.env }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- range $key, $secret := $.Values.envSecrets }} + - name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ $secret }} + key: {{ $key | quote }} + {{- end }} + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumeMounts: + - name: data + mountPath: /data/grafana + {{- range $path, $bytes := .Files.Glob "dashboards/*.json" }} + {{- $filename := trimSuffix (ext $path) (base $path) }} + - name: dashboards-istio-{{ $filename }} + mountPath: "/var/lib/grafana/dashboards/istio/{{ base $path }}" + subPath: {{ base $path }} + readOnly: true + {{- end }} + - name: config + mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" + subPath: datasources.yaml + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" + subPath: dashboardproviders.yaml + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} + volumes: + - name: config + configMap: + name: istio-grafana + - name: data +{{- if .Values.persist }} + persistentVolumeClaim: + claimName: istio-grafana-pvc +{{- else }} + emptyDir: {} +{{- end }} +{{- range $path, $bytes := .Files.Glob "dashboards/*.json" }} +{{- $filename := trimSuffix (ext $path) (base $path) }} + - name: dashboards-istio-{{ $filename }} + configMap: + name: istio-grafana-configuration-dashboards-{{ $filename }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/grafana-ports-mtls.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/grafana-ports-mtls.yaml new file mode 100755 index 000000000..b9a392651 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/grafana-ports-mtls.yaml @@ -0,0 +1,17 @@ +{{ define "grafana-default.yaml.tpl" }} +apiVersion: authentication.istio.io/v1alpha1 +kind: Policy +metadata: + name: grafana-ports-mtls-disabled + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + targets: + - name: grafana + ports: + - number: {{ .Values.service.externalPort }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/ingress.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/ingress.yaml new file mode 100755 index 000000000..0ebe71f61 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: grafana + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: +{{- if .Values.ingress.hosts }} + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ if $.Values.contextPath }} {{ $.Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: grafana + servicePort: 3000 + {{- end -}} +{{- else }} + - http: + paths: + - path: {{ if .Values.contextPath }} {{ .Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: grafana + servicePort: 3000 +{{- end }} + {{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/pvc.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/pvc.yaml new file mode 100755 index 000000000..e376a13a5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/pvc.yaml @@ -0,0 +1,19 @@ +{{- if .Values.persist }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: istio-grafana-pvc + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + storageClassName: {{ .Values.storageClassName }} + accessModes: + - {{ .Values.accessMode }} + resources: + requests: + storage: 5Gi +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/service.yaml new file mode 100755 index 000000000..1dfd82c33 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/service.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: {{ .Release.Namespace }} + annotations: + {{- range $key, $val := .Values.service.annotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + labels: + app: {{ template "grafana.name" . }} + chart: {{ template "grafana.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.externalPort }} + targetPort: 3000 + protocol: TCP + name: {{ .Values.service.name }} + selector: + app: grafana +{{- if .Values.service.loadBalancerIP }} + loadBalancerIP: "{{ .Values.service.loadBalancerIP }}" +{{- end }} + {{if .Values.service.loadBalancerSourceRanges}} + loadBalancerSourceRanges: + {{range $rangeList := .Values.service.loadBalancerSourceRanges}} + - {{ $rangeList }} + {{end}} + {{end}} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/tests/test-grafana-connection.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/tests/test-grafana-connection.yaml new file mode 100755 index 000000000..e9268c4b9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/templates/tests/test-grafana-connection.yaml @@ -0,0 +1,37 @@ +{{- if .Values.global.enableHelmTest }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "grafana.fullname" . }}-test + namespace: {{ .Release.Namespace }} + labels: + app: grafana-test + chart: {{ template "grafana.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: grafana + annotations: + sidecar.istio.io/inject: "false" + helm.sh/hook: test-success +spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: "{{ template "grafana.fullname" . }}-test" + image: pstauffer/curl:v1.0.3 + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" + command: ['curl'] + args: ['http://grafana:{{ .Values.grafana.service.externalPort }}'] + restartPolicy: Never + affinity: + {{- include "nodeaffinity" . | indent 4 }} + {{- include "podAntiAffinity" . | indent 4 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 2 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 2 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/grafana/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/grafana/values.yaml new file mode 100755 index 000000000..d6c192a38 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/grafana/values.yaml @@ -0,0 +1,117 @@ +# +# addon grafana configuration +# +enabled: false +replicaCount: 1 +image: + repository: grafana/grafana + tag: 6.1.6 +ingress: + enabled: false + ## Used to create an Ingress record. + hosts: + - grafana.local + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + tls: + # Secrets must be manually created in the namespace. + # - secretName: grafana-tls + # hosts: + # - grafana.local +persist: false +storageClassName: "" +accessMode: ReadWriteMany +security: + enabled: false + secretName: grafana + usernameKey: username + passphraseKey: passphrase +nodeSelector: {} +tolerations: [] + +env: {} + # Define additional environment variables for configuring grafana. + # @see https://grafana.com/docs/installation/configuration/#using-environment-variables + # Format: env_variable_name: value + # For example: + # GF_SMTP_ENABLED: true + # GF_SMTP_HOST: email-smtp.eu-west-1.amazonaws.com:2587 + # GF_SMTP_FROM_ADDRESS: alerts@mydomain.com + # GF_SMTP_FROM_NAME: Grafana + +envSecrets: {} + # The key name and ENV name must match in the secrets file. + # @see https://grafana.com/docs/installation/configuration/#using-environment-variables + # For example: + # --- + # apiVersion: v1 + # kind: Secret + # metadata: + # name: grafana-secrets + # namespace: istio-system + # data: + # GF_SMTP_USER: bXl1c2Vy + # GF_SMTP_PASSWORD: bXlwYXNzd29yZA== + # type: Opaque + # --- + # env_variable_key_name: secretsName + # --- + # GF_SMTP_USER: grafana-secrets + # GF_SMTP_PASSWORD: grafana-secrets + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +contextPath: /grafana +service: + annotations: {} + name: http + type: ClusterIP + externalPort: 3000 + loadBalancerIP: + loadBalancerSourceRanges: + +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + orgId: 1 + url: http://prometheus:9090 + access: proxy + isDefault: true + jsonData: + timeInterval: 5s + editable: true + +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'istio' + orgId: 1 + folder: 'istio' + type: file + disableDeletion: false + options: + path: /var/lib/grafana/dashboards/istio diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/Chart.yaml new file mode 100755 index 000000000..1a87643a6 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +appVersion: "0.1" +description: Istio CoreDNS provides DNS resolution for services in multicluster setups. +name: istiocoredns +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/_helpers.tpl new file mode 100755 index 000000000..e7add11bb --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "istiocoredns.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "istiocoredns.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "istiocoredns.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrole.yaml new file mode 100755 index 000000000..4242a327f --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrole.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istiocoredns + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["networking.istio.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..bafd0ca3b --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-istiocoredns-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istiocoredns +subjects: +- kind: ServiceAccount + name: istiocoredns-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/configmap.yaml new file mode 100755 index 000000000..50d166fe5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/configmap.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + Corefile: | + .:53 { + errors + health + proxy global 127.0.0.1:8053 { + protocol grpc insecure + } + prometheus :9153 + proxy . /etc/resolv.conf + cache 30 + reload + } +--- diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/deployment.yaml new file mode 100755 index 000000000..4cd5f002d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istiocoredns + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: istiocoredns + strategy: + rollingUpdate: + maxSurge: {{ .Values.rollingMaxSurge }} + maxUnavailable: {{ .Values.rollingMaxUnavailable }} + template: + metadata: + name: istiocoredns + labels: + app: istiocoredns + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istiocoredns-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: coredns + image: {{ .Values.coreDNSImage }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + - name: istio-coredns-plugin + command: + - /usr/local/bin/plugin + image: {{ .Values.coreDNSPluginImage }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: 8053 + name: dns-grpc + protocol: TCP + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/service.yaml new file mode 100755 index 000000000..a6311017c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: istiocoredns + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + app: istiocoredns + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/serviceaccount.yaml new file mode 100755 index 000000000..e2627cf45 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istiocoredns-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istiocoredns.name" . }} + chart: {{ template "istiocoredns.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/values.yaml new file mode 100755 index 000000000..4724df7e5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/istiocoredns/values.yaml @@ -0,0 +1,35 @@ +# +# addon istiocoredns tracing configuration +# +enabled: false +replicaCount: 1 +rollingMaxSurge: 100% +rollingMaxUnavailable: 25% +coreDNSImage: coredns/coredns:1.1.2 +# Source code for the plugin can be found at +# https://github.com/istio-ecosystem/istio-coredns-plugin +# The plugin listens for DNS requests from coredns server at 127.0.0.1:8053 +coreDNSPluginImage: istio/coredns-plugin:0.2-istio-1.1 +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/Chart.yaml new file mode 100755 index 000000000..a0db26f1e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Kiali is an open source project for service mesh observability, refer + to https://www.kiali.io for details. +name: kiali +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/_helpers.tpl new file mode 100755 index 000000000..6b0095769 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "kiali.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "kiali.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kiali.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrole.yaml new file mode 100755 index 000000000..d9091a0b9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrole.yaml @@ -0,0 +1,267 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kiali + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: [""] + resources: + - configmaps + - endpoints + - namespaces + - nodes + - pods + - pods/log + - replicationcontrollers + - services + verbs: + - get + - list + - watch +- apiGroups: ["extensions", "apps"] + resources: + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch +- apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiGroups: ["config.istio.io"] + resources: + - adapters + - apikeys + - bypasses + - authorizations + - checknothings + - circonuses + - cloudwatches + - deniers + - dogstatsds + - edges + - fluentds + - handlers + - instances + - kubernetesenvs + - kuberneteses + - listcheckers + - listentries + - logentries + - memquotas + - metrics + - noops + - opas + - prometheuses + - quotas + - quotaspecbindings + - quotaspecs + - rbacs + - redisquotas + - reportnothings + - rules + - signalfxs + - solarwindses + - stackdrivers + - statsds + - stdios + - templates + - tracespans + - zipkins + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: ["networking.istio.io"] + resources: + - destinationrules + - gateways + - serviceentries + - sidecars + - virtualservices + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: ["authentication.istio.io"] + resources: + - meshpolicies + - policies + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: ["rbac.istio.io"] + resources: + - clusterrbacconfigs + - rbacconfigs + - servicerolebindings + - serviceroles + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: ["monitoring.kiali.io"] + resources: + - monitoringdashboards + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kiali-viewer + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: [""] + resources: + - configmaps + - endpoints + - namespaces + - nodes + - pods + - pods/log + - replicationcontrollers + - services + verbs: + - get + - list + - watch +- apiGroups: ["extensions", "apps"] + resources: + - deployments + - replicasets + - statefulsets + verbs: + - get + - list + - watch +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch +- apiGroups: ["batch"] + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch +- apiGroups: ["config.istio.io"] + resources: + - adapters + - apikeys + - bypasses + - authorizations + - checknothings + - circonuses + - cloudwatches + - deniers + - dogstatsds + - edges + - fluentds + - handlers + - instances + - kubernetesenvs + - kuberneteses + - listcheckers + - listentries + - logentries + - memquotas + - metrics + - noops + - opas + - prometheuses + - quotas + - quotaspecbindings + - quotaspecs + - rbacs + - redisquotas + - reportnothings + - rules + - signalfxs + - solarwindses + - stackdrivers + - statsds + - stdios + - templates + - tracespans + - zipkins + verbs: + - get + - list + - watch +- apiGroups: ["networking.istio.io"] + resources: + - destinationrules + - gateways + - serviceentries + - sidecars + - virtualservices + verbs: + - get + - list + - watch +- apiGroups: ["authentication.istio.io"] + resources: + - meshpolicies + - policies + verbs: + - get + - list + - watch +- apiGroups: ["rbac.istio.io"] + resources: + - clusterrbacconfigs + - rbacconfigs + - servicerolebindings + - serviceroles + verbs: + - get + - list + - watch +- apiGroups: ["monitoring.kiali.io"] + resources: + - monitoringdashboards + verbs: + - get + - list diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..881796716 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/clusterrolebinding.yaml @@ -0,0 +1,37 @@ +{{- if not .Values.dashboard.viewOnlyMode }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-kiali-admin-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiali +subjects: +- kind: ServiceAccount + name: kiali-service-account + namespace: {{ .Release.Namespace }} +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-kiali-viewer-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kiali-viewer +subjects: +- kind: ServiceAccount + name: kiali-service-account + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/configmap.yaml new file mode 100755 index 000000000..cd63a90cc --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: kiali + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + config.yaml: | + istio_namespace: {{ .Release.Namespace }} + auth: + strategy: {{ .Values.dashboard.auth.strategy }} + server: + port: 20001 +{{- if .Values.contextPath }} + web_root: {{ .Values.contextPath }} +{{- end }} + external_services: + tracing: + url: {{ .Values.dashboard.jaegerURL }} + grafana: + url: {{ .Values.dashboard.grafanaURL }} + prometheus: + url: {{ .Values.prometheusAddr }} +{{- if .Values.security.enabled }} + identity: + cert_file: {{ .Values.security.cert_file }} + private_key_file: {{ .Values.security.private_key_file }} +{{- end}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/demosecret.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/demosecret.yaml new file mode 100755 index 000000000..ad44298c3 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/demosecret.yaml @@ -0,0 +1,16 @@ +{{- if .Values.createDemoSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.dashboard.secretName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +type: Opaque +data: + username: YWRtaW4= # admin + passphrase: YWRtaW4= # admin +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/deployment.yaml new file mode 100755 index 000000000..9389f58d0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/deployment.yaml @@ -0,0 +1,100 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kiali + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: kiali + template: + metadata: + name: kiali + labels: + app: kiali + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + scheduler.alpha.kubernetes.io/critical-pod: "" + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + kiali.io/runtimes: go,kiali + spec: + serviceAccountName: kiali-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - image: "{{ .Values.hub }}/{{ .Values.image }}:{{ .Values.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + name: kiali + command: + - "/opt/kiali/kiali" + - "-config" + - "/kiali-configuration/config.yaml" + - "-v" + - "3" + readinessProbe: + httpGet: + path: {{ .Values.contextPath }}/healthz + port: 20001 + scheme: {{ if .Values.security.enabled }} 'HTTPS' {{ else }} 'HTTP' {{ end }} + initialDelaySeconds: 5 + periodSeconds: 30 + livenessProbe: + httpGet: + path: {{ .Values.contextPath }}/healthz + port: 20001 + scheme: {{ if .Values.security.enabled }} 'HTTPS' {{ else }} 'HTTP' {{ end }} + initialDelaySeconds: 5 + periodSeconds: 30 + env: + - name: ACTIVE_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: kiali-configuration + mountPath: "/kiali-configuration" + - name: kiali-cert + mountPath: "/kiali-cert" + - name: kiali-secret + mountPath: "/kiali-secret" + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + volumes: + - name: kiali-configuration + configMap: + name: kiali + - name: kiali-cert + secret: + secretName: istio.kiali-service-account +{{- if not .Values.security.enabled }} + optional: true +{{- end }} + - name: kiali-secret + secret: + secretName: {{ .Values.dashboard.secretName }} + optional: true + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/ingress.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/ingress.yaml new file mode 100755 index 000000000..2e2a0de3a --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: kiali + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: +{{- if .Values.ingress.hosts }} + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ if $.Values.contextPath }} {{ $.Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: kiali + servicePort: 20001 + {{- end -}} +{{- else }} + - http: + paths: + - path: {{ if .Values.contextPath }} {{ .Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: kiali + servicePort: 20001 +{{- end }} + {{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/service.yaml new file mode 100755 index 000000000..1aa79bfdb --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: kiali + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + ports: + - name: http-kiali + protocol: TCP + port: 20001 + selector: + app: kiali diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/serviceaccount.yaml new file mode 100755 index 000000000..2ae38a1ab --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: kiali-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "kiali.name" . }} + chart: {{ template "kiali.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/tests/test-kiali-connection.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/tests/test-kiali-connection.yaml new file mode 100755 index 000000000..d798f7f42 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/templates/tests/test-kiali-connection.yaml @@ -0,0 +1,37 @@ +{{- if .Values.global.enableHelmTest }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "kiali.fullname" . }}-test + namespace: {{ .Release.Namespace }} + labels: + app: kiali-test + chart: {{ template "kiali.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: kiali + annotations: + sidecar.istio.io/inject: "false" + helm.sh/hook: test-success +spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: "{{ template "kiali.fullname" . }}-test" + image: pstauffer/curl:v1.0.3 + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" + command: ['curl'] + args: ['http://kiali:20001'] + restartPolicy: Never + affinity: + {{- include "nodeaffinity" . | indent 4 }} + {{- include "podAntiAffinity" . | indent 4 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 2 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 2 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/kiali/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/kiali/values.yaml new file mode 100755 index 000000000..bb536f1c8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/kiali/values.yaml @@ -0,0 +1,63 @@ +# +# addon kiali +# +enabled: false # Note that if using the demo or demo-auth yaml when installing via Helm, this default will be `true`. +replicaCount: 1 +hub: quay.io/kiali +image: kiali +tag: v1.4 +contextPath: /kiali # The root context path to access the Kiali UI. +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +ingress: + enabled: false + ## Used to create an Ingress record. + hosts: + - kiali.local + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + tls: + # Secrets must be manually created in the namespace. + # - secretName: kiali-tls + # hosts: + # - kiali.local + +dashboard: + auth: + strategy: login # Can be anonymous, login, or openshift + secretName: kiali # You must create a secret with this name - one is not provided out-of-box. + viewOnlyMode: false # Bind the service account to a role with only read access + grafanaURL: # If you have Grafana installed and it is accessible to client browsers, then set this to its external URL. Kiali will redirect users to this URL when Grafana metrics are to be shown. + jaegerURL: # If you have Jaeger installed and it is accessible to client browsers, then set this property to its external URL. Kiali will redirect users to this URL when Jaeger tracing is to be shown. +prometheusAddr: http://prometheus:9090 + +# When true, a secret will be created with a default username and password. Useful for demos. +createDemoSecret: false + +security: + enabled: false + cert_file: /kiali-cert/cert-chain.pem + private_key_file: /kiali-cert/key.pem diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/Chart.yaml new file mode 100755 index 000000000..1c7617ee0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for mixer deployment +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- mixer +name: mixer +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/_helpers.tpl new file mode 100755 index 000000000..dac6da036 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "mixer.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mixer.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "mixer.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml new file mode 100755 index 000000000..377b47d03 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/autoscale.yaml @@ -0,0 +1,29 @@ +{{- range $key, $spec := .Values }} +{{- if or (eq $key "policy") (eq $key "telemetry") }} +{{- if and $spec.enabled $spec.autoscaleEnabled $spec.autoscaleMin $spec.autoscaleMax }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: istio-{{ $key }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ template "mixer.name" $ }} + chart: {{ template "mixer.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} +spec: + maxReplicas: {{ $spec.autoscaleMax }} + minReplicas: {{ $spec.autoscaleMin }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: istio-{{ $key }} + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ $spec.cpu.targetAverageUtilization }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrole.yaml new file mode 100755 index 000000000..3d7438f2d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrole.yaml @@ -0,0 +1,24 @@ +{{- if or (.Values.policy.enabled) (.Values.telemetry.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-mixer-{{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["config.istio.io"] # istio CRD watcher + resources: ["*"] + verbs: ["create", "get", "list", "watch", "patch"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps", "endpoints", "pods", "services", "namespaces", "secrets", "replicationcontrollers"] + verbs: ["get", "list", "watch"] +- apiGroups: ["extensions", "apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..773e68b34 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/clusterrolebinding.yaml @@ -0,0 +1,19 @@ +{{- if or (.Values.policy.enabled) (.Values.telemetry.enabled) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-mixer-admin-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-mixer-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-mixer-service-account + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/config.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/config.yaml new file mode 100755 index 000000000..43b2c5ced --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/config.yaml @@ -0,0 +1,1088 @@ +{{- if or (.Values.policy.enabled) (.Values.telemetry.enabled) }} +apiVersion: "config.istio.io/v1alpha2" +kind: attributemanifest +metadata: + name: istioproxy + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + attributes: + origin.ip: + valueType: IP_ADDRESS + origin.uid: + valueType: STRING + origin.user: + valueType: STRING + request.headers: + valueType: STRING_MAP + request.id: + valueType: STRING + request.host: + valueType: STRING + request.method: + valueType: STRING + request.path: + valueType: STRING + request.url_path: + valueType: STRING + request.query_params: + valueType: STRING_MAP + request.reason: + valueType: STRING + request.referer: + valueType: STRING + request.scheme: + valueType: STRING + request.total_size: + valueType: INT64 + request.size: + valueType: INT64 + request.time: + valueType: TIMESTAMP + request.useragent: + valueType: STRING + response.code: + valueType: INT64 + response.duration: + valueType: DURATION + response.headers: + valueType: STRING_MAP + response.total_size: + valueType: INT64 + response.size: + valueType: INT64 + response.time: + valueType: TIMESTAMP + response.grpc_status: + valueType: STRING + response.grpc_message: + valueType: STRING + source.uid: + valueType: STRING + source.user: # DEPRECATED + valueType: STRING + source.principal: + valueType: STRING + destination.uid: + valueType: STRING + destination.principal: + valueType: STRING + destination.port: + valueType: INT64 + connection.event: + valueType: STRING + connection.id: + valueType: STRING + connection.received.bytes: + valueType: INT64 + connection.received.bytes_total: + valueType: INT64 + connection.sent.bytes: + valueType: INT64 + connection.sent.bytes_total: + valueType: INT64 + connection.duration: + valueType: DURATION + connection.mtls: + valueType: BOOL + connection.requested_server_name: + valueType: STRING + context.protocol: + valueType: STRING + context.proxy_error_code: + valueType: STRING + context.timestamp: + valueType: TIMESTAMP + context.time: + valueType: TIMESTAMP + # Deprecated, kept for compatibility + context.reporter.local: + valueType: BOOL + context.reporter.kind: + valueType: STRING + context.reporter.uid: + valueType: STRING + api.service: + valueType: STRING + api.version: + valueType: STRING + api.operation: + valueType: STRING + api.protocol: + valueType: STRING + request.auth.principal: + valueType: STRING + request.auth.audiences: + valueType: STRING + request.auth.presenter: + valueType: STRING + request.auth.claims: + valueType: STRING_MAP + request.auth.raw_claims: + valueType: STRING + request.api_key: + valueType: STRING + rbac.permissive.response_code: + valueType: STRING + rbac.permissive.effective_policy_id: + valueType: STRING + check.error_code: + valueType: INT64 + check.error_message: + valueType: STRING + check.cache_hit: + valueType: BOOL + quota.cache_hit: + valueType: BOOL + context.proxy_version: + valueType: STRING + +--- +apiVersion: "config.istio.io/v1alpha2" +kind: attributemanifest +metadata: + name: kubernetes + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + attributes: + source.ip: + valueType: IP_ADDRESS + source.labels: + valueType: STRING_MAP + source.metadata: + valueType: STRING_MAP + source.name: + valueType: STRING + source.namespace: + valueType: STRING + source.owner: + valueType: STRING + source.serviceAccount: + valueType: STRING + source.services: + valueType: STRING + source.workload.uid: + valueType: STRING + source.workload.name: + valueType: STRING + source.workload.namespace: + valueType: STRING + destination.ip: + valueType: IP_ADDRESS + destination.labels: + valueType: STRING_MAP + destination.metadata: + valueType: STRING_MAP + destination.owner: + valueType: STRING + destination.name: + valueType: STRING + destination.container.name: + valueType: STRING + destination.namespace: + valueType: STRING + destination.service.uid: + valueType: STRING + destination.service.name: + valueType: STRING + destination.service.namespace: + valueType: STRING + destination.service.host: + valueType: STRING + destination.serviceAccount: + valueType: STRING + destination.workload.uid: + valueType: STRING + destination.workload.name: + valueType: STRING + destination.workload.namespace: + valueType: STRING +--- +{{- if and .Values.adapters.stdio.enabled .Values.telemetry.enabled }} +apiVersion: "config.istio.io/v1alpha2" +kind: handler +metadata: + name: stdio + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledAdapter: stdio + params: + outputAsJson: {{ .Values.adapters.stdio.outputAsJson }} +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: accesslog + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: logentry + params: + severity: '"Info"' + timestamp: request.time + variables: + sourceIp: source.ip | ip("0.0.0.0") + sourceApp: source.labels["app"] | "" + sourcePrincipal: source.principal | "" + sourceName: source.name | "" + sourceWorkload: source.workload.name | "" + sourceNamespace: source.namespace | "" + sourceOwner: source.owner | "" + destinationApp: destination.labels["app"] | "" + destinationIp: destination.ip | ip("0.0.0.0") + destinationServiceHost: destination.service.host | request.host | "" + destinationWorkload: destination.workload.name | "" + destinationName: destination.name | "" + destinationNamespace: destination.namespace | "" + destinationOwner: destination.owner | "" + destinationPrincipal: destination.principal | "" + apiClaims: request.auth.raw_claims | "" + apiKey: request.api_key | request.headers["x-api-key"] | "" + protocol: request.scheme | context.protocol | "http" + method: request.method | "" + url: request.path | "" + responseCode: response.code | 0 + responseFlags: context.proxy_error_code | "" + responseSize: response.size | 0 + permissiveResponseCode: rbac.permissive.response_code | "none" + permissiveResponsePolicyID: rbac.permissive.effective_policy_id | "none" + requestSize: request.size | 0 + requestId: request.headers["x-request-id"] | "" + clientTraceId: request.headers["x-client-trace-id"] | "" + latency: response.duration | "0ms" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + requestedServerName: connection.requested_server_name | "" + userAgent: request.useragent | "" + responseTimestamp: response.time + receivedBytes: request.total_size | 0 + sentBytes: response.total_size | 0 + referer: request.referer | "" + httpAuthority: request.headers[":authority"] | request.host | "" + xForwardedFor: request.headers["x-forwarded-for"] | "0.0.0.0" + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + grpcStatus: response.grpc_status | "" + grpcMessage: response.grpc_message | "" + monitored_resource_type: '"global"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: tcpaccesslog + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: logentry + params: + severity: '"Info"' + timestamp: context.time | timestamp("2017-01-01T00:00:00Z") + variables: + connectionEvent: connection.event | "" + sourceIp: source.ip | ip("0.0.0.0") + sourceApp: source.labels["app"] | "" + sourcePrincipal: source.principal | "" + sourceName: source.name | "" + sourceWorkload: source.workload.name | "" + sourceNamespace: source.namespace | "" + sourceOwner: source.owner | "" + destinationApp: destination.labels["app"] | "" + destinationIp: destination.ip | ip("0.0.0.0") + destinationServiceHost: destination.service.host | "" + destinationWorkload: destination.workload.name | "" + destinationName: destination.name | "" + destinationNamespace: destination.namespace | "" + destinationOwner: destination.owner | "" + destinationPrincipal: destination.principal | "" + protocol: context.protocol | "tcp" + connectionDuration: connection.duration | "0ms" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + requestedServerName: connection.requested_server_name | "" + receivedBytes: connection.received.bytes | 0 + sentBytes: connection.sent.bytes | 0 + totalReceivedBytes: connection.received.bytes_total | 0 + totalSentBytes: connection.sent.bytes_total | 0 + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + responseFlags: context.proxy_error_code | "" + monitored_resource_type: '"global"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: stdio + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "http" || context.protocol == "grpc" + actions: + - handler: stdio + instances: + - accesslog +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: stdiotcp + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "tcp" + actions: + - handler: stdio + instances: + - tcpaccesslog +{{- end }} +--- +{{- if and .Values.adapters.prometheus.enabled .Values.telemetry.enabled }} +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: requestcount + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: "1" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | request.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + response_flags: context.proxy_error_code | "-" + permissive_response_code: rbac.permissive.response_code | "none" + permissive_response_policyid: rbac.permissive.effective_policy_id | "none" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: requestduration + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: response.duration | "0ms" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | request.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + response_flags: context.proxy_error_code | "-" + permissive_response_code: rbac.permissive.response_code | "none" + permissive_response_policyid: rbac.permissive.effective_policy_id | "none" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: requestsize + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: request.size | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | request.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + response_flags: context.proxy_error_code | "-" + permissive_response_code: rbac.permissive.response_code | "none" + permissive_response_policyid: rbac.permissive.effective_policy_id | "none" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: responsesize + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: response.size | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | request.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + request_protocol: api.protocol | context.protocol | "unknown" + response_code: response.code | 200 + response_flags: context.proxy_error_code | "-" + permissive_response_code: rbac.permissive.response_code | "none" + permissive_response_policyid: rbac.permissive.effective_policy_id | "none" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: tcpbytesent + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: connection.sent.bytes | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + response_flags: context.proxy_error_code | "-" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: tcpbytereceived + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: connection.received.bytes | 0 + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + response_flags: context.proxy_error_code | "-" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: tcpconnectionsopened + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: "1" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + response_flags: context.proxy_error_code | "-" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: tcpconnectionsclosed + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: metric + params: + value: "1" + dimensions: + reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") + source_workload: source.workload.name | "unknown" + source_workload_namespace: source.workload.namespace | "unknown" + source_principal: source.principal | "unknown" + source_app: source.labels["app"] | "unknown" + source_version: source.labels["version"] | "unknown" + destination_workload: destination.workload.name | "unknown" + destination_workload_namespace: destination.workload.namespace | "unknown" + destination_principal: destination.principal | "unknown" + destination_app: destination.labels["app"] | "unknown" + destination_version: destination.labels["version"] | "unknown" + destination_service: destination.service.host | "unknown" + destination_service_name: destination.service.name | "unknown" + destination_service_namespace: destination.service.namespace | "unknown" + connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) + response_flags: context.proxy_error_code | "-" + monitored_resource_type: '"UNSPECIFIED"' +--- +apiVersion: "config.istio.io/v1alpha2" +kind: handler +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledAdapter: prometheus + params: + metricsExpirationPolicy: + metricsExpiryDuration: "{{ .Values.adapters.prometheus.metricsExpiryDuration }}" + metrics: + - name: requests_total + instance_name: requestcount.instance.{{ .Release.Namespace }} + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - response_flags + - permissive_response_code + - permissive_response_policyid + - connection_security_policy + - name: request_duration_seconds + instance_name: requestduration.instance.{{ .Release.Namespace }} + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - response_flags + - permissive_response_code + - permissive_response_policyid + - connection_security_policy + buckets: + explicit_buckets: + bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] + - name: request_bytes + instance_name: requestsize.instance.{{ .Release.Namespace }} + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - response_flags + - permissive_response_code + - permissive_response_policyid + - connection_security_policy + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: response_bytes + instance_name: responsesize.instance.{{ .Release.Namespace }} + kind: DISTRIBUTION + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - request_protocol + - response_code + - response_flags + - permissive_response_code + - permissive_response_policyid + - connection_security_policy + buckets: + exponentialBuckets: + numFiniteBuckets: 8 + scale: 1 + growthFactor: 10 + - name: tcp_sent_bytes_total + instance_name: tcpbytesent.instance.{{ .Release.Namespace }} + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy + - response_flags + - name: tcp_received_bytes_total + instance_name: tcpbytereceived.instance.{{ .Release.Namespace }} + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy + - response_flags + - name: tcp_connections_opened_total + instance_name: tcpconnectionsopened.instance.{{ .Release.Namespace }} + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy + - response_flags + - name: tcp_connections_closed_total + instance_name: tcpconnectionsclosed.instance.{{ .Release.Namespace }} + kind: COUNTER + label_names: + - reporter + - source_app + - source_principal + - source_workload + - source_workload_namespace + - source_version + - destination_app + - destination_principal + - destination_workload + - destination_workload_namespace + - destination_version + - destination_service + - destination_service_name + - destination_service_namespace + - connection_security_policy + - response_flags +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promhttp + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: (context.protocol == "http" || context.protocol == "grpc") && (match((request.useragent | "-"), "kube-probe*") == false) && (match((request.useragent | "-"), "Prometheus*") == false) + actions: + - handler: prometheus + instances: + - requestcount + - requestduration + - requestsize + - responsesize +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promtcp + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "tcp" + actions: + - handler: prometheus + instances: + - tcpbytesent + - tcpbytereceived +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promtcpconnectionopen + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "tcp" && ((connection.event | "na") == "open") + actions: + - handler: prometheus + instances: + - tcpconnectionsopened +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: promtcpconnectionclosed + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "tcp" && ((connection.event | "na") == "close") + actions: + - handler: prometheus + instances: + - tcpconnectionsclosed +{{- end }} +--- +{{- if and .Values.adapters.kubernetesenv.enabled (or .Values.policy.enabled .Values.telemetry.enabled) }} +apiVersion: "config.istio.io/v1alpha2" +kind: handler +metadata: + name: kubernetesenv + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledAdapter: kubernetesenv + params: + # when running from mixer root, use the following config after adding a + # symbolic link to a kubernetes config file via: + # + # $ ln -s ~/.kube/config mixer/adapter/kubernetes/kubeconfig + # + # kubeconfig_path: "mixer/adapter/kubernetes/kubeconfig" + +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: kubeattrgenrulerule + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + actions: + - handler: kubernetesenv + instances: + - attributes +--- +apiVersion: "config.istio.io/v1alpha2" +kind: rule +metadata: + name: tcpkubeattrgenrulerule + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + match: context.protocol == "tcp" + actions: + - handler: kubernetesenv + instances: + - attributes +--- +apiVersion: "config.istio.io/v1alpha2" +kind: instance +metadata: + name: attributes + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + compiledTemplate: kubernetes + params: + # Pass the required attribute data to the adapter + source_uid: source.uid | "" + source_ip: source.ip | ip("0.0.0.0") # default to unspecified ip addr + destination_uid: destination.uid | "" + destination_port: destination.port | 0 + attributeBindings: + # Fill the new attributes from the adapter produced output. + # $out refers to an instance of OutputTemplate message + source.ip: $out.source_pod_ip | ip("0.0.0.0") + source.uid: $out.source_pod_uid | "unknown" + source.labels: $out.source_labels | emptyStringMap() + source.name: $out.source_pod_name | "unknown" + source.namespace: $out.source_namespace | "default" + source.owner: $out.source_owner | "unknown" + source.serviceAccount: $out.source_service_account_name | "unknown" + source.workload.uid: $out.source_workload_uid | "unknown" + source.workload.name: $out.source_workload_name | "unknown" + source.workload.namespace: $out.source_workload_namespace | "unknown" + destination.ip: $out.destination_pod_ip | ip("0.0.0.0") + destination.uid: $out.destination_pod_uid | "unknown" + destination.labels: $out.destination_labels | emptyStringMap() + destination.name: $out.destination_pod_name | "unknown" + destination.container.name: $out.destination_container_name | "unknown" + destination.namespace: $out.destination_namespace | "default" + destination.owner: $out.destination_owner | "unknown" + destination.serviceAccount: $out.destination_service_account_name | "unknown" + destination.workload.uid: $out.destination_workload_uid | "unknown" + destination.workload.name: $out.destination_workload_name | "unknown" + destination.workload.namespace: $out.destination_workload_namespace | "unknown" +{{- end }} +--- +{{- if .Values.policy.enabled }} +# Configuration needed by Mixer. +# Mixer cluster is delivered via CDS +# Specify mixer cluster settings +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: istio-policy + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: istio-policy.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + {{- if .Values.global.defaultConfigVisibilitySettings }} + exportTo: + - '*' + {{- end }} + trafficPolicy: + {{- if .Values.global.controlPlaneSecurityEnabled }} + portLevelSettings: + - port: + number: 15004 + tls: + mode: ISTIO_MUTUAL + {{- end}} + connectionPool: + http: + http2MaxRequests: 10000 + maxRequestsPerConnection: 10000 +{{- end }} +--- +{{- if .Values.telemetry.enabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: istio-telemetry + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: istio-telemetry.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + {{- if .Values.global.defaultConfigVisibilitySettings }} + exportTo: + - '*' + {{- end }} + trafficPolicy: + {{- if .Values.global.controlPlaneSecurityEnabled }} + portLevelSettings: + - port: + number: 15004 + tls: + mode: ISTIO_MUTUAL + {{- end}} + connectionPool: + http: + http2MaxRequests: 10000 + maxRequestsPerConnection: 10000 +{{- end }} +--- +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/deployment.yaml new file mode 100755 index 000000000..2b32b1e1d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/deployment.yaml @@ -0,0 +1,418 @@ +{{- define "policy_container" }} + spec: + serviceAccountName: istio-mixer-service-account +{{- if $.Values.global.priorityClassName }} + priorityClassName: "{{ $.Values.global.priorityClassName }}" +{{- end }} + volumes: + - name: istio-certs + secret: + secretName: istio.istio-mixer-service-account + optional: true + {{- if $.Values.global.sds.enabled }} + - hostPath: + path: /var/run/sds + name: sds-uds-path + - name: istio-token + projected: + sources: + - serviceAccountToken: + audience: {{ $.Values.global.sds.token.aud }} + expirationSeconds: 43200 + path: istio-token + {{- end }} + - name: uds-socket + emptyDir: {} + - name: policy-adapter-secret + secret: + secretName: policy-adapter-secret + optional: true + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} + containers: + - name: mixer +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $.Values.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.global.monitoringPort }} + - containerPort: 42422 + args: + - --monitoringPort={{ .Values.global.monitoringPort }} + - --address + - unix:///sock/mixer.socket +{{- if $.Values.global.logging.level }} + - --log_output_level={{ $.Values.global.logging.level }} +{{- end}} +{{- if $.Values.global.useMCP }} + {{- if $.Values.global.controlPlaneSecurityEnabled}} + - --configStoreURL=mcps://istio-galley.{{ $.Release.Namespace }}.svc:9901 + {{- else }} + - --configStoreURL=mcp://istio-galley.{{ $.Release.Namespace }}.svc:9901 + {{- end }} +{{- else }} + - --configStoreURL=k8s:// +{{- end }} + - --configDefaultNamespace={{ $.Release.Namespace }} + {{- if $.Values.adapters.useAdapterCRDs }} + - --useAdapterCRDs=true + {{- else }} + - --useAdapterCRDs=false + {{- end }} + - --useTemplateCRDs=false + {{- if $.Values.global.tracer.zipkin.address }} + - --trace_zipkin_url=http://{{- $.Values.global.tracer.zipkin.address }}/api/v1/spans + {{- else }} + - --trace_zipkin_url=http://zipkin.{{ $.Release.Namespace }}:9411/api/v1/spans + {{- end }} + {{- if .Values.env }} + env: + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} + resources: +{{- if .Values.policy.resources }} +{{ toYaml .Values.policy.resources | indent 10 }} +{{- else if .Values.resources }} +{{ toYaml .Values.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + volumeMounts: +{{- if $.Values.global.useMCP }} + - name: istio-certs + mountPath: /etc/certs + readOnly: true +{{- end }} + - name: uds-socket + mountPath: /sock + livenessProbe: + httpGet: + path: /version + port: {{ .Values.global.monitoringPort }} + initialDelaySeconds: 5 + periodSeconds: 5 + - name: istio-proxy +{{- if contains "/" $.Values.global.proxy.image }} + image: "{{ $.Values.global.proxy.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $.Values.global.proxy.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + ports: + - containerPort: 9091 + - containerPort: 15004 + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - --domain + - $(POD_NAMESPACE).svc.{{ $.Values.global.proxy.clusterDomain }} + - --serviceCluster + - istio-policy + - --templateFile + - /etc/istio/proxy/envoy_policy.yaml.tmpl + {{- if $.Values.global.controlPlaneSecurityEnabled }} + - --controlPlaneAuthPolicy + - MUTUAL_TLS + {{- else }} + - --controlPlaneAuthPolicy + - NONE + {{- end }} + {{- if $.Values.global.trustDomain }} + - --trust-domain={{ $.Values.global.trustDomain }} + {{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: SDS_ENABLED + value: "{{ $.Values.global.sds.enabled }}" + resources: +{{- if $.Values.global.proxy.resources }} +{{ toYaml $.Values.global.proxy.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + {{- if $.Values.global.sds.enabled }} + - name: sds-uds-path + mountPath: /var/run/sds + readOnly: true + - name: istio-token + mountPath: /var/run/secrets/tokens + {{- end }} + - name: uds-socket + mountPath: /sock + - name: policy-adapter-secret + mountPath: /var/run/secrets/istio.io/policy/adapter + readOnly: true +{{- end }} + +{{- define "telemetry_container" }} + spec: + serviceAccountName: istio-mixer-service-account +{{- if $.Values.global.priorityClassName }} + priorityClassName: "{{ $.Values.global.priorityClassName }}" +{{- end }} + volumes: + - name: istio-certs + secret: + secretName: istio.istio-mixer-service-account + optional: true + {{- if $.Values.global.sds.enabled }} + - hostPath: + path: /var/run/sds + name: sds-uds-path + - name: istio-token + projected: + sources: + - serviceAccountToken: + audience: {{ $.Values.global.sds.token.aud }} + expirationSeconds: 43200 + path: istio-token + {{- end }} + - name: uds-socket + emptyDir: {} + - name: telemetry-adapter-secret + secret: + secretName: telemetry-adapter-secret + optional: true + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} + containers: + - name: mixer +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $.Values.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.global.monitoringPort }} + - containerPort: 42422 + args: + - --monitoringPort={{ .Values.global.monitoringPort }} + - --address + - unix:///sock/mixer.socket +{{- if $.Values.global.logging.level }} + - --log_output_level={{ $.Values.global.logging.level }} +{{- end}} +{{- if $.Values.global.useMCP }} + {{- if $.Values.global.controlPlaneSecurityEnabled}} + - --configStoreURL=mcps://istio-galley.{{ $.Release.Namespace }}.svc:9901 + - --certFile=/etc/certs/cert-chain.pem + - --keyFile=/etc/certs/key.pem + - --caCertFile=/etc/certs/root-cert.pem + {{- else }} + - --configStoreURL=mcp://istio-galley.{{ $.Release.Namespace }}.svc:9901 + {{- end }} +{{- else }} + - --configStoreURL=k8s:// +{{- end }} + - --configDefaultNamespace={{ $.Release.Namespace }} + {{- if $.Values.adapters.useAdapterCRDs }} + - --useAdapterCRDs=true + {{- else }} + - --useAdapterCRDs=false + {{- end }} + {{- if $.Values.global.tracer.zipkin.address }} + - --trace_zipkin_url=http://{{- $.Values.global.tracer.zipkin.address }}/api/v1/spans + {{- else }} + - --trace_zipkin_url=http://zipkin.{{ $.Release.Namespace }}:9411/api/v1/spans + {{- end }} + - --averageLatencyThreshold + - {{ $.Values.telemetry.loadshedding.latencyThreshold }} + - --loadsheddingMode + - {{ $.Values.telemetry.loadshedding.mode }} + {{- if .Values.env }} + env: + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} + resources: +{{- if .Values.telemetry.resources }} +{{ toYaml .Values.telemetry.resources | indent 10 }} +{{- else if .Values.resources }} +{{ toYaml .Values.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + volumeMounts: +{{- if $.Values.global.useMCP }} + - name: istio-certs + mountPath: /etc/certs + readOnly: true +{{- end }} + - name: telemetry-adapter-secret + mountPath: /var/run/secrets/istio.io/telemetry/adapter + readOnly: true + - name: uds-socket + mountPath: /sock + livenessProbe: + httpGet: + path: /version + port: {{ .Values.global.monitoringPort }} + initialDelaySeconds: 5 + periodSeconds: 5 + - name: istio-proxy +{{- if contains "/" $.Values.global.proxy.image }} + image: "{{ $.Values.global.proxy.image }}" +{{- else }} + image: "{{ $.Values.global.hub }}/{{ $.Values.global.proxy.image }}:{{ $.Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ $.Values.global.imagePullPolicy }} + ports: + - containerPort: 9091 + - containerPort: 15004 + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - --domain + - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} + - --serviceCluster + - istio-telemetry + - --templateFile + - /etc/istio/proxy/envoy_telemetry.yaml.tmpl + {{- if $.Values.global.controlPlaneSecurityEnabled }} + - --controlPlaneAuthPolicy + - MUTUAL_TLS + {{- else }} + - --controlPlaneAuthPolicy + - NONE + {{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: SDS_ENABLED + value: "{{ $.Values.global.sds.enabled }}" + resources: +{{- if $.Values.global.proxy.resources }} +{{ toYaml $.Values.global.proxy.resources | indent 10 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 10 }} +{{- end }} + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + {{- if $.Values.global.sds.enabled }} + - name: sds-uds-path + mountPath: /var/run/sds + readOnly: true + - name: istio-token + mountPath: /var/run/secrets/tokens + {{- end }} + - name: uds-socket + mountPath: /sock +{{- end }} + + +{{- range $key, $spec := .Values }} +{{- if or (eq $key "policy") (eq $key "telemetry") }} +{{- if $spec.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-{{ $key }} + namespace: {{ $.Release.Namespace }} + labels: + app: istio-mixer + chart: {{ template "mixer.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + istio: mixer +spec: +{{- if not $spec.autoscaleEnabled }} +{{- if $spec.replicaCount }} + replicas: {{ $spec.replicaCount }} +{{- else }} + replicas: 1 +{{- end }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: {{ $spec.rollingMaxSurge }} + maxUnavailable: {{ $spec.rollingMaxUnavailable }} + selector: + matchLabels: + istio: mixer + istio-mixer-type: {{ $key }} + template: + metadata: + labels: + app: {{ $key }} + chart: {{ template "mixer.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + istio: mixer + istio-mixer-type: {{ $key }} + annotations: + sidecar.istio.io/inject: "false" +{{- with $.Values.podAnnotations }} +{{ toYaml . | indent 8 }} +{{- end }} +{{- if eq $key "policy"}} +{{- template "policy_container" $ }} +{{- else }} +{{- template "telemetry_container" $ }} +{{- end }} + +--- +{{- end }} +{{- end }} +{{- end }} {{/* range */}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..a6bfe8668 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/poddisruptionbudget.yaml @@ -0,0 +1,32 @@ +{{- range $key, $spec := .Values }} +{{- if or (eq $key "policy") (eq $key "telemetry") }} +{{- if $spec.enabled }} +{{- if $.Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: istio-{{ $key }} + namespace: {{ $.Release.Namespace }} + labels: + app: {{ $key }} + chart: {{ template "mixer.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + version: {{ $.Chart.Version }} + istio: mixer + istio-mixer-type: {{ $key }} +spec: +{{- if $.Values.global.defaultPodDisruptionBudget.enabled }} +{{ include "podDisruptionBudget.spec" $.Values.global.defaultPodDisruptionBudget }} +{{- end }} + selector: + matchLabels: + app: {{ $key }} + release: {{ $.Release.Name }} + istio: mixer + istio-mixer-type: {{ $key }} +--- +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/service.yaml new file mode 100755 index 000000000..79cc4a582 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/service.yaml @@ -0,0 +1,39 @@ +{{- range $key, $spec := .Values }} +{{- if or (eq $key "policy") (eq $key "telemetry") }} +{{- if $spec.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: istio-{{ $key }} + namespace: {{ $.Release.Namespace }} + annotations: + networking.istio.io/exportTo: "*" + labels: + app: {{ template "mixer.name" $ }} + chart: {{ template "mixer.chart" $ }} + heritage: {{ $.Release.Service }} + release: {{ $.Release.Name }} + istio: mixer +spec: + ports: + - name: grpc-mixer + port: 9091 + - name: grpc-mixer-mtls + port: 15004 + - name: http-monitoring + port: {{ $.Values.global.monitoringPort }} +{{- if eq $key "telemetry" }} + - name: prometheus + port: 42422 +{{- if $spec.sessionAffinityEnabled }} + sessionAffinity: ClientIP +{{- end }} +{{- end }} + selector: + istio: mixer + istio-mixer-type: {{ $key }} +--- +{{- end }} +{{- end }} +{{- end }} + diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/serviceaccount.yaml new file mode 100755 index 000000000..9d3da7dd6 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/templates/serviceaccount.yaml @@ -0,0 +1,18 @@ +{{- if or (.Values.policy.enabled) (.Values.telemetry.enabled) }} +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-mixer-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "mixer.name" . }} + chart: {{ template "mixer.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/mixer/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/mixer/values.yaml new file mode 100755 index 000000000..4d9452356 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/mixer/values.yaml @@ -0,0 +1,99 @@ +# +# mixer configuration +# +image: mixer + +env: + GODEBUG: gctrace=1 + # max procs should be ceil(cpu limit + 1) + GOMAXPROCS: "6" + +policy: + # if policy is enabled, global.disablePolicyChecks has affect. + enabled: false + replicaCount: 1 + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + cpu: + targetAverageUtilization: 80 + rollingMaxSurge: 100% + rollingMaxUnavailable: 25% + +telemetry: + enabled: true + replicaCount: 1 + autoscaleEnabled: true + autoscaleMin: 1 + autoscaleMax: 5 + cpu: + targetAverageUtilization: 80 + rollingMaxSurge: 100% + rollingMaxUnavailable: 25% + sessionAffinityEnabled: false + + # mixer load shedding configuration. + # When mixer detects that it is overloaded, it starts rejecting grpc requests. + loadshedding: + # disabled, logonly or enforce + mode: enforce + # based on measurements 100ms p50 translates to p99 of under 1s. This is ok for telemetry which is inherently async. + latencyThreshold: 100ms + resources: + requests: + cpu: 1000m + memory: 1G + limits: + # It is best to do horizontal scaling of mixer using moderate cpu allocation. + # We have experimentally found that these values work well. + cpu: 4800m + memory: 4G + + # Set reportBatchMaxEntries to 0 to use the default batching behavior (i.e., every 100 requests). + # A positive value indicates the number of requests that are batched before telemetry data + # is sent to the mixer server + reportBatchMaxEntries: 100 + + # Set reportBatchMaxTime to 0 to use the default batching behavior (i.e., every 1 second). + # A positive time value indicates the maximum wait time since the last request will telemetry data + # be batched before being sent to the mixer server + reportBatchMaxTime: 1s + +podAnnotations: {} +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +adapters: + kubernetesenv: + enabled: true + + # stdio is a debug adapter in istio-telemetry, it is not recommended for production use. + stdio: + enabled: false + outputAsJson: true + prometheus: + enabled: true + metricsExpiryDuration: 10m + # Setting this to false sets the useAdapterCRDs mixer startup argument to false + useAdapterCRDs: false diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/Chart.yaml new file mode 100755 index 000000000..8a952758c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for nodeagent deployment +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- nodeagent +name: nodeagent +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/_helpers.tpl new file mode 100755 index 000000000..fda6043d0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "nodeagent.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nodeagent.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nodeagent.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrole.yaml new file mode 100755 index 000000000..9127b05e3 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrole.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-nodeagent-{{ .Release.Namespace }} + labels: + app: {{ template "nodeagent.name" . }} + chart: {{ template "nodeagent.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..963757e72 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-nodeagent-{{ .Release.Namespace }} + labels: + app: {{ template "nodeagent.name" . }} + chart: {{ template "nodeagent.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-nodeagent-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-nodeagent-service-account + namespace: {{ .Release.Namespace }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/daemonset.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/daemonset.yaml new file mode 100755 index 000000000..00935f9f7 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/daemonset.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: istio-nodeagent + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "nodeagent.name" . }} + chart: {{ template "nodeagent.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: nodeagent +spec: + selector: + matchLabels: + istio: nodeagent + template: + metadata: + labels: + app: {{ template "nodeagent.name" . }} + chart: {{ template "nodeagent.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: nodeagent + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-nodeagent-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: nodeagent +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + volumeMounts: + - mountPath: /var/run/sds + name: sdsudspath + env: + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} + - name: "TRUST_DOMAIN" + value: "{{ .Values.global.trustDomain }}" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumes: + - name: sdsudspath + hostPath: + path: /var/run/sds + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} + updateStrategy: + type: RollingUpdate \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/serviceaccount.yaml new file mode 100755 index 000000000..b52f852d8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-nodeagent-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "nodeagent.name" . }} + chart: {{ template "nodeagent.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/values.yaml new file mode 100755 index 000000000..76c5503af --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/nodeagent/values.yaml @@ -0,0 +1,35 @@ +# +# nodeagent configuration +# +enabled: false +image: node-agent-k8s +env: + # name of authentication provider. + CA_PROVIDER: "" + # CA endpoint. + CA_ADDR: "" + # names of authentication provider's plugins. + PLUGINS: "" +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/Chart.yaml new file mode 100755 index 000000000..099d16615 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for pilot deployment +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- pilot +name: pilot +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/_helpers.tpl new file mode 100755 index 000000000..c812c3709 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "pilot.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "pilot.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "pilot.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml new file mode 100755 index 000000000..1a9945136 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/autoscale.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.autoscaleEnabled .Values.autoscaleMin .Values.autoscaleMax }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + maxReplicas: {{ .Values.autoscaleMax }} + minReplicas: {{ .Values.autoscaleMin }} + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: istio-pilot + metrics: + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.cpu.targetAverageUtilization }} +--- +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrole.yaml new file mode 100755 index 000000000..0435c3ebd --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrole.yaml @@ -0,0 +1,34 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-pilot-{{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["config.istio.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["rbac.istio.io"] + resources: ["*"] + verbs: ["get", "watch", "list"] +- apiGroups: ["networking.istio.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["authentication.istio.io"] + resources: ["*"] + verbs: ["*"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["*"] +- apiGroups: ["extensions"] + resources: ["ingresses", "ingresses/status"] + verbs: ["*"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["endpoints", "pods", "services", "namespaces", "nodes", "secrets"] + verbs: ["get", "list", "watch"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..ef9281ca8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-pilot-{{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-pilot-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-pilot-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/deployment.yaml new file mode 100755 index 000000000..e43ea7d9d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/deployment.yaml @@ -0,0 +1,223 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} + # TODO: default template doesn't have this, which one is right ? + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: pilot + annotations: + checksum/config-volume: {{ template "istio.configmap.checksum" . }} +spec: +{{- if not .Values.autoscaleEnabled }} +{{- if .Values.replicaCount }} + replicas: {{ .Values.replicaCount }} +{{- else }} + replicas: 1 +{{- end }} +{{- end }} + strategy: + rollingUpdate: + maxSurge: {{ .Values.rollingMaxSurge }} + maxUnavailable: {{ .Values.rollingMaxUnavailable }} + selector: + matchLabels: + istio: pilot + template: + metadata: + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: pilot + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-pilot-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: discovery +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + - "discovery" + - --monitoringAddr=:{{ .Values.global.monitoringPort }} +{{- if $.Values.global.logging.level }} + - --log_output_level={{ $.Values.global.logging.level }} +{{- end}} + - --domain + - {{ .Values.global.proxy.clusterDomain }} +{{- if .Values.global.oneNamespace }} + - "-a" + - {{ .Release.Namespace }} +{{- end }} +{{- if and $.Values.global.controlPlaneSecurityEnabled (not .Values.sidecar)}} + - --secureGrpcAddr + - ":15011" +{{- else }} + - --secureGrpcAddr + - "" +{{- end }} +{{- if .Values.global.trustDomain }} + - --trust-domain={{ .Values.global.trustDomain }} +{{- end }} + - --keepaliveMaxServerConnectionAge + - "{{ .Values.keepaliveMaxServerConnectionAge }}" + ports: + - containerPort: 8080 + - containerPort: 15010 +{{- if not .Values.sidecar }} + - containerPort: 15011 +{{- end }} + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 30 + timeoutSeconds: 5 + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- if .Values.env }} + {{- range $key, $val := .Values.env }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} +{{- if .Values.traceSampling }} + - name: PILOT_TRACE_SAMPLING + value: "{{ .Values.traceSampling }}" +{{- end }} + - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND + value: "{{ .Values.enableProtocolSniffingForOutbound }}" + - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND + value: "{{ .Values.enableProtocolSniffingForInbound }}" + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumeMounts: + - name: config-volume + mountPath: /etc/istio/config + - name: istio-certs + mountPath: /etc/certs + readOnly: true +{{- if .Values.sidecar }} + - name: istio-proxy +{{- if contains "/" .Values.global.proxy.image }} + image: "{{ .Values.global.proxy.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: 15003 + - containerPort: 15005 + - containerPort: 15007 + - containerPort: 15011 + args: + - proxy + - --domain + - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} + - --serviceCluster + - istio-pilot + - --templateFile + - /etc/istio/proxy/envoy_pilot.yaml.tmpl + {{- if $.Values.global.controlPlaneSecurityEnabled}} + - --controlPlaneAuthPolicy + - MUTUAL_TLS + {{- else }} + - --controlPlaneAuthPolicy + - NONE + {{- end }} + {{- if .Values.global.trustDomain }} + - --trust-domain={{ .Values.global.trustDomain }} + {{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: SDS_ENABLED + value: "{{ $.Values.global.sds.enabled }}" + resources: +{{- if .Values.global.proxy.resources }} +{{ toYaml .Values.global.proxy.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumeMounts: + - name: istio-certs + mountPath: /etc/certs + readOnly: true + {{- if $.Values.global.sds.enabled }} + - name: sds-uds-path + mountPath: /var/run/sds + readOnly: true + - name: istio-token + mountPath: /var/run/secrets/tokens + {{- end }} +{{- end }} + volumes: + {{- if $.Values.global.sds.enabled }} + - hostPath: + path: /var/run/sds + name: sds-uds-path + - name: istio-token + projected: + sources: + - serviceAccountToken: + audience: {{ $.Values.global.sds.token.aud }} + expirationSeconds: 43200 + path: istio-token + {{- end }} + - name: config-volume + configMap: + name: istio + - name: istio-certs + secret: + secretName: istio.istio-pilot-service-account + optional: true + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/meshexpansion.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/meshexpansion.yaml new file mode 100755 index 000000000..4f3d59570 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/meshexpansion.yaml @@ -0,0 +1,91 @@ +{{- if .Values.global.meshExpansion.enabled }} +{{- if .Values.global.meshExpansion.useILB }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: meshexpansion-ilb-vs-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + hosts: + - istio-pilot.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + gateways: + - meshexpansion-ilb-gateway + tcp: + - match: + - port: 15011 + route: + - destination: + host: istio-pilot.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 15011 + - match: + - port: 15010 + route: + - destination: + host: istio-pilot.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 15010 + - match: + - port: 5353 + route: + - destination: + host: kube-dns.kube-system.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 53 +--- +{{- else }} + +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: meshexpansion-vs-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + hosts: + - istio-pilot.{{ $.Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + gateways: + - meshexpansion-gateway + tcp: + - match: + - port: 15011 + route: + - destination: + host: istio-pilot.{{ $.Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 15011 +--- +{{- end }} + +{{- if .Values.global.controlPlaneSecurityEnabled }} +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: meshexpansion-dr-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: istio-pilot.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + trafficPolicy: + portLevelSettings: + - port: + number: 15011 + tls: + mode: DISABLE +--- +{{- end }} +{{- end }} + diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..fd9e06a71 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/poddisruptionbudget.yaml @@ -0,0 +1,22 @@ +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: pilot +spec: +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +{{ include "podDisruptionBudget.spec" .Values.global.defaultPodDisruptionBudget }} +{{- end }} + selector: + matchLabels: + app: {{ template "pilot.name" . }} + release: {{ .Release.Name }} + istio: pilot +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/service.yaml new file mode 100755 index 000000000..a61d93025 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: pilot +spec: + ports: + - port: 15010 + name: grpc-xds # direct + - port: 15011 + name: https-xds # mTLS + - port: 8080 + name: http-legacy-discovery # direct + - port: {{ .Values.global.monitoringPort }} + name: http-monitoring + selector: + istio: pilot diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/serviceaccount.yaml new file mode 100755 index 000000000..7ec2a66de --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-pilot-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "pilot.name" . }} + chart: {{ template "pilot.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/pilot/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/pilot/values.yaml new file mode 100755 index 000000000..58e2a0357 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/pilot/values.yaml @@ -0,0 +1,56 @@ +# +# pilot configuration +# +enabled: true +autoscaleEnabled: true +autoscaleMin: 1 +autoscaleMax: 5 +# specify replicaCount when autoscaleEnabled: false +# replicaCount: 1 +rollingMaxSurge: 100% +rollingMaxUnavailable: 25% +image: pilot +sidecar: true +traceSampling: 1.0 +# if protocol sniffing is enabled for outbound +enableProtocolSniffingForOutbound: true +# if protocol sniffing is enabled for inbound +enableProtocolSniffingForInbound: false +# Resources for a small pilot install +resources: + requests: + cpu: 500m + memory: 2048Mi +env: + PILOT_PUSH_THROTTLE: 100 + GODEBUG: gctrace=1 +cpu: + targetAverageUtilization: 80 +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +# The following is used to limit how long a sidecar can be connected +# to a pilot. It balances out load across pilot instances at the cost of +# increasing system churn. +keepaliveMaxServerConnectionAge: 30m diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/Chart.yaml new file mode 100755 index 000000000..8327144e0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +appVersion: 2.8.0 +description: A Helm chart for Kubernetes +name: prometheus +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/_helpers.tpl new file mode 100755 index 000000000..039388329 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "prometheus.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "prometheus.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "prometheus.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrole.yaml new file mode 100755 index 000000000..06fdfaf53 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrole.yaml @@ -0,0 +1,24 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus-{{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + - nodes/proxy + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrolebindings.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrolebindings.yaml new file mode 100755 index 000000000..295e0df72 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/clusterrolebindings.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus-{{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-{{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: prometheus + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/configmap.yaml new file mode 100755 index 000000000..1b26fa5a1 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/configmap.yaml @@ -0,0 +1,281 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + prometheus.yml: |- + global: + scrape_interval: {{ .Values.scrapeInterval }} + scrape_configs: + + - job_name: 'istio-mesh' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-telemetry;prometheus + + # Scrape config for envoy stats + - job_name: 'envoy-stats' + metrics_path: /stats/prometheus + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_container_port_name] + action: keep + regex: '.*-envoy-prom' + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:15090 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name + + - job_name: 'istio-policy' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-policy;http-monitoring + + - job_name: 'istio-telemetry' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-telemetry;http-monitoring + + - job_name: 'pilot' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-pilot;http-monitoring + + - job_name: 'galley' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-galley;http-monitoring + + - job_name: 'citadel' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - {{ .Release.Namespace }} + + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: istio-citadel;http-monitoring + + # scrape config for API servers + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - default + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kubernetes;https + + # scrape config for nodes (kubelet) + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + + # Scrape config for Kubelet cAdvisor. + # + # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics + # (those whose names begin with 'container_') have been removed from the + # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to + # retrieve those metrics. + # + # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor + # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" + # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with + # the --cadvisor-port=0 Kubelet flag). + # + # This job is not necessary and should be removed in Kubernetes 1.6 and + # earlier versions, or it will cause the metrics to be scraped twice. + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + + # scrape config for service endpoints. + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name + + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: # If first two labels are present, pod should be scraped by the istio-secure job. + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + # Keep target if there's no sidecar or if prometheus.io/scheme is explicitly set to "http" + - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: keep + regex: ((;.*)|(.*;http)) + - source_labels: [__meta_kubernetes_pod_annotation_istio_mtls] + action: drop + regex: (true) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name + + - job_name: 'kubernetes-pods-istio-secure' + scheme: https + tls_config: + ca_file: /etc/istio-certs/root-cert.pem + cert_file: /etc/istio-certs/cert-chain.pem + key_file: /etc/istio-certs/key.pem + insecure_skip_verify: true # prometheus does not support secure naming. + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + # sidecar status annotation is added by sidecar injector and + # istio_workload_mtls_ability can be specifically placed on a pod to indicate its ability to receive mtls traffic. + - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_istio_mtls] + action: keep + regex: (([^;]+);([^;]*))|(([^;]*);(true)) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme] + action: drop + regex: (http) + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__] # Only keep address that is host:port + action: keep # otherwise an extra target with ':443' is added for https scheme + regex: ([^:]+):(\d+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: pod_name \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/deployment.yaml new file mode 100755 index 000000000..64214e0de --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/deployment.yaml @@ -0,0 +1,80 @@ +# TODO: the original template has service account, roles, etc +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: prometheus +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: prometheus + image: "{{ .Values.hub }}/{{ .Values.image }}:{{ .Values.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + - '--storage.tsdb.retention={{ .Values.retention }}' + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - containerPort: 9090 + name: http + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - mountPath: /etc/istio-certs + name: istio-certs + volumes: + - name: config-volume + configMap: + name: prometheus + - name: istio-certs + secret: + defaultMode: 420 +{{- if not .Values.security.enabled }} + optional: true +{{- end }} + secretName: istio.default + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/ingress.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/ingress.yaml new file mode 100755 index 000000000..43be65523 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/ingress.yaml @@ -0,0 +1,40 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: +{{- if .Values.ingress.hosts }} + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ if $.Values.contextPath }} {{ $.Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: prometheus + servicePort: 9090 + {{- end -}} +{{- else }} + - http: + paths: + - path: {{ if .Values.contextPath }} {{ .Values.contextPath }} {{ else }} / {{ end }} + backend: + serviceName: prometheus + servicePort: 9090 +{{- end }} + {{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/service.yaml new file mode 100755 index 000000000..d92525df0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/service.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + annotations: + prometheus.io/scrape: 'true' + {{- range $key, $val := .Values.service.annotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + app: prometheus + ports: + - name: http-prometheus + protocol: TCP + port: 9090 + +{{- if .Values.service.nodePort.enabled }} +# Using separate ingress for nodeport, to avoid conflict with pilot e2e test configs. +--- +apiVersion: v1 +kind: Service +metadata: + name: prometheus-nodeport + namespace: {{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + type: NodePort + ports: + - port: 9090 + nodePort: {{ .Values.service.nodePort.port }} + name: http-prometheus + selector: + app: prometheus +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/serviceaccount.yaml new file mode 100755 index 000000000..7c2fab3f4 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: prometheus + namespace: {{ .Release.Namespace }} + labels: + app: prometheus + chart: {{ template "prometheus.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/tests/test-prometheus-connection.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/tests/test-prometheus-connection.yaml new file mode 100755 index 000000000..45b025edf --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/templates/tests/test-prometheus-connection.yaml @@ -0,0 +1,36 @@ +{{- if .Values.global.enableHelmTest }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "prometheus.fullname" . }}-test + namespace: {{ .Release.Namespace }} + labels: + app: prometheus-test + chart: {{ template "prometheus.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: prometheus + annotations: + sidecar.istio.io/inject: "false" + helm.sh/hook: test-success +spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: "{{ template "prometheus.fullname" . }}-test" + image: pstauffer/curl:v1.0.3 + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" + command: ['sh', '-c', 'for i in 1 2 3; do curl http://prometheus:9090/-/ready && exit 0 || sleep 15; done; exit 1'] + restartPolicy: Never + affinity: + {{- include "nodeaffinity" . | indent 4 }} + {{- include "podAntiAffinity" . | indent 4 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 2 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 2 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/prometheus/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/values.yaml new file mode 100755 index 000000000..c16f99a62 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/prometheus/values.yaml @@ -0,0 +1,60 @@ +# +# addon prometheus configuration +# +enabled: true +replicaCount: 1 +hub: docker.io/prom +image: prometheus +tag: v2.8.0 +retention: 6h +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +# Controls the frequency of prometheus scraping +scrapeInterval: 15s + +contextPath: /prometheus + +ingress: + enabled: false + ## Used to create an Ingress record. + hosts: + - prometheus.local + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + tls: + # Secrets must be manually created in the namespace. + # - secretName: prometheus-tls + # hosts: + # - prometheus.local + +service: + annotations: {} + nodePort: + enabled: false + port: 32090 + +security: + enabled: true diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/Chart.yaml new file mode 100755 index 000000000..61b0b2ac3 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for istio authentication +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- security +name: security +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/_helpers.tpl new file mode 100755 index 000000000..7f36f9d51 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "security.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "security.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "security.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrole.yaml new file mode 100755 index 000000000..75f2dec65 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrole.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-citadel-{{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["create", "get", "update"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "watch", "list", "update", "delete"] +- apiGroups: [""] + resources: ["serviceaccounts", "services", "namespaces"] + verbs: ["get", "watch", "list"] +- apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..0a15799ce --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-citadel-{{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-citadel-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-citadel-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/configmap.yaml new file mode 100755 index 000000000..14749fd65 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/configmap.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-security-custom-resources + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel +data: + custom-resources.yaml: |- + {{- if .Values.global.mtls.enabled }} + {{- include "security-default.yaml.tpl" . | indent 4}} + {{- else }} + {{- include "security-permissive.yaml.tpl" . | indent 4}} + {{- end }} + run.sh: |- + {{- include "install-custom-resources.sh.tpl" . | indent 4}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/create-custom-resources-job.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/create-custom-resources-job.yaml new file mode 100755 index 000000000..a9658bbce --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/create-custom-resources-job.yaml @@ -0,0 +1,107 @@ +{{- if .Values.createMeshPolicy }} +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-security-post-install-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: istio-security-post-install-{{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +rules: +- apiGroups: ["authentication.istio.io"] # needed to create default authn policy + resources: ["*"] + verbs: ["*"] +- apiGroups: ["networking.istio.io"] # needed to create security destination rules + resources: ["*"] + verbs: ["*"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["get"] +- apiGroups: ["extensions", "apps"] + resources: ["deployments", "replicasets"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: istio-security-post-install-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-security-post-install-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-security-post-install-account + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: istio-security-post-install-{{ .Values.global.tag | printf "%v" | trunc 32 }} + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-delete-policy": hook-succeeded + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + template: + metadata: + name: istio-security-post-install + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + serviceAccountName: istio-security-post-install-account + containers: + - name: kubectl + image: "{{ .Values.global.hub }}/kubectl:{{ .Values.global.tag }}" + imagePullPolicy: IfNotPresent + command: [ "/bin/bash", "/tmp/security/run.sh", "/tmp/security/custom-resources.yaml" ] + volumeMounts: + - mountPath: "/tmp/security" + name: tmp-configmap-security + volumes: + - name: tmp-configmap-security + configMap: + name: istio-security-custom-resources + restartPolicy: OnFailure + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/deployment.yaml new file mode 100755 index 000000000..7b43e9a61 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/deployment.yaml @@ -0,0 +1,114 @@ +# istio CA watching all namespaces +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-citadel + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + istio: citadel + strategy: + rollingUpdate: + maxSurge: {{ .Values.rollingMaxSurge }} + maxUnavailable: {{ .Values.rollingMaxUnavailable }} + template: + metadata: + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-citadel-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: citadel +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + {{- if .Values.global.sds.enabled }} + - --sds-enabled=true + {{- end }} + - --append-dns-names=true + - --grpc-port=8060 + - --citadel-storage-namespace={{ .Release.Namespace }} + - --custom-dns-names=istio-pilot-service-account.{{ .Release.Namespace }}:istio-pilot.{{ .Release.Namespace }} + - --monitoring-port={{ .Values.global.monitoringPort }} + {{- if .Values.selfSigned }} + - --self-signed-ca=true + {{- else }} + - --self-signed-ca=false + - --signing-cert=/etc/cacerts/ca-cert.pem + - --signing-key=/etc/cacerts/ca-key.pem + - --root-cert=/etc/cacerts/root-cert.pem + - --cert-chain=/etc/cacerts/cert-chain.pem + {{- end }} + {{- if .Values.global.trustDomain }} + - --trust-domain={{ .Values.global.trustDomain }} + {{- end }} + {{- if .Values.workloadCertTtl }} + - --workload-cert-ttl={{ .Values.workloadCertTtl }} + {{- end }} + {{- if .Values.citadelHealthCheck }} + - --liveness-probe-path=/tmp/ca.liveness # path to the liveness health check status file + - --liveness-probe-interval=60s # interval for health check file update + - --probe-check-interval=15s # interval for health status check + {{- end }} + env: + - name: CITADEL_ENABLE_NAMESPACES_BY_DEFAULT + value: "{{ .Values.enableNamespacesByDefault }}" + {{- if .Values.citadelHealthCheck }} + livenessProbe: + exec: + command: + - /usr/local/bin/istio_ca + - probe + - --probe-path=/tmp/ca.liveness # path to the liveness health check status file + - --interval=125s # the maximum time gap allowed between the file mtime and the current sys clock + initialDelaySeconds: 60 + periodSeconds: 60 + {{- end }} + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} +{{- if not .Values.selfSigned }} + volumeMounts: + - name: cacerts + mountPath: /etc/cacerts + readOnly: true + volumes: + - name: cacerts + secret: + secretName: cacerts + optional: true +{{- end }} + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-mtls.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-mtls.yaml new file mode 100755 index 000000000..75e4a18e3 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-mtls.yaml @@ -0,0 +1,63 @@ +{{- define "security-default.yaml.tpl" }} +# These policy and destination rules effectively enable mTLS for all services in the mesh. For now, +# they are added to Istio installation yaml for backward compatible. In future, they should be in +# a separated yaml file so that customer can enable mTLS independent from installation. + +# Authentication policy to enable mutual TLS for all services (that have sidecar) in the mesh. +apiVersion: "authentication.istio.io/v1alpha1" +kind: "MeshPolicy" +metadata: + name: "default" + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + peers: + - mtls: {} +--- +# Corresponding destination rule to configure client side to use mutual TLS when talking to +# any service (host) in the mesh. +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: "default" + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: "*.local" + {{- if .Values.global.defaultConfigVisibilitySettings }} + exportTo: + - '*' + {{- end }} + trafficPolicy: + tls: + mode: ISTIO_MUTUAL +--- +# Destination rule to disable (m)TLS when talking to API server, as API server doesn't have sidecar. +# Customer should add similar destination rules for other services that don't have sidecar. +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: "api-server" + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + host: "kubernetes.default.svc.{{ .Values.global.proxy.clusterDomain }}" + {{- if .Values.global.defaultConfigVisibilitySettings }} + exportTo: + - '*' + {{- end }} + trafficPolicy: + tls: + mode: DISABLE +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-permissive.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-permissive.yaml new file mode 100755 index 000000000..a6931b3b9 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/enable-mesh-permissive.yaml @@ -0,0 +1,16 @@ +{{- define "security-permissive.yaml.tpl" }} +# Authentication policy to enable permissive mode for all services (that have sidecar) in the mesh. +apiVersion: "authentication.istio.io/v1alpha1" +kind: "MeshPolicy" +metadata: + name: "default" + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + peers: + - mtls: + mode: PERMISSIVE +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/meshexpansion.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/meshexpansion.yaml new file mode 100755 index 000000000..581ce964a --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/meshexpansion.yaml @@ -0,0 +1,56 @@ +{{- if .Values.global.meshExpansion.enabled }} +{{- if .Values.global.meshExpansion.useILB }} +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: meshexpansion-vs-citadel-ilb + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel +spec: + hosts: + - istio-citadel.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + gateways: + - meshexpansion-ilb-gateway + tcp: + - match: + - port: 8060 + route: + - destination: + host: istio-citadel.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 8060 +--- +{{- else }} + +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: meshexpansion-vs-citadel + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel +spec: + hosts: + - istio-citadel.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + gateways: + - meshexpansion-gateway + tcp: + - match: + - port: 8060 + route: + - destination: + host: istio-citadel.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }} + port: + number: 8060 +--- +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/service.yaml new file mode 100755 index 000000000..efea17544 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + # we use the normal name here (e.g. 'prometheus') + # as grafana is configured to use this as a data source + name: istio-citadel + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: citadel +spec: + ports: + - name: grpc-citadel + port: 8060 + targetPort: 8060 + protocol: TCP + - name: http-monitoring + port: {{ .Values.global.monitoringPort }} + selector: + istio: citadel diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/serviceaccount.yaml new file mode 100755 index 000000000..d07d566fa --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/serviceaccount.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-citadel-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "security.name" . }} + chart: {{ template "security.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/templates/tests/test-citadel-connection.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/tests/test-citadel-connection.yaml new file mode 100755 index 000000000..6fc742a6e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/templates/tests/test-citadel-connection.yaml @@ -0,0 +1,36 @@ +{{- if .Values.global.enableHelmTest }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ template "security.fullname" . }}-test + namespace: {{ .Release.Namespace }} + labels: + app: istio-citadel-test + chart: {{ template "security.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + istio: citadel + annotations: + sidecar.istio.io/inject: "false" + helm.sh/hook: test-success +spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: "{{ template "security.fullname" . }}-test" + image: pstauffer/curl:v1.0.3 + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" + command: ['sh', '-c', 'for i in 1 2 3; do curl http://istio-citadel:{{ .Values.global.monitoringPort }}/version && exit 0 || sleep 15; done; exit 1'] + restartPolicy: Never + affinity: + {{- include "nodeaffinity" . | indent 4 }} + {{- include "podAntiAffinity" . | indent 4 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 2 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 2 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/security/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/security/values.yaml new file mode 100755 index 000000000..6bbb43401 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/security/values.yaml @@ -0,0 +1,48 @@ +# +# security configuration +# +enabled: true +replicaCount: 1 +rollingMaxSurge: 100% +rollingMaxUnavailable: 25% +image: citadel +selfSigned: true # indicate if self-signed CA is used. +createMeshPolicy: true +nodeSelector: {} +tolerations: [] +# Enable health checking on the Citadel CSR signing API. +# https://istio.io/docs/tasks/security/health-check/ +citadelHealthCheck: false +# 90*24hour = 2160h +workloadCertTtl: 2160h + +# Determines Citadel default behavior if the ca.istio.io/env or ca.istio.io/override +# labels are not found on a given namespace. +# +# For example: consider a namespace called "target", which has neither the "ca.istio.io/env" +# nor the "ca.istio.io/override" namespace labels. To decide whether or not to generate secrets +# for service accounts created in this "target" namespace, Citadel will defer to this option. If the value +# of this option is "true" in this case, secrets will be generated for the "target" namespace. +# If the value of this option is "false" Citadel will not generate secrets upon service account creation. +enableNamespacesByDefault: true + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/Chart.yaml new file mode 100755 index 000000000..f8f9037df --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +appVersion: 1.3.3 +description: Helm chart for sidecar injector webhook deployment +engine: gotpl +icon: https://istio.io/favicons/android-192x192.png +keywords: +- istio +- sidecarInjectorWebhook +name: sidecarInjectorWebhook +sources: +- http://github.com/istio/istio +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/_helpers.tpl new file mode 100755 index 000000000..f3b9fb15b --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "sidecar-injector.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sidecar-injector.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sidecar-injector.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrole.yaml new file mode 100755 index 000000000..27f9acb51 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrole.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: istio-sidecar-injector-{{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations"] + verbs: ["get", "list", "watch", "patch"] diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..748a93244 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-sidecar-injector-admin-role-binding-{{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-sidecar-injector-{{ .Release.Namespace }} +subjects: + - kind: ServiceAccount + name: istio-sidecar-injector-service-account + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/deployment.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/deployment.yaml new file mode 100755 index 000000000..abbb93d39 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/deployment.yaml @@ -0,0 +1,110 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-sidecar-injector + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + istio: sidecar-injector + strategy: + rollingUpdate: + maxSurge: {{ .Values.rollingMaxSurge }} + maxUnavailable: {{ .Values.rollingMaxUnavailable }} + template: + metadata: + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector + annotations: + sidecar.istio.io/inject: "false" + spec: + serviceAccountName: istio-sidecar-injector-service-account +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: sidecar-injector-webhook +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.image }}:{{ .Values.global.tag }}" +{{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + args: + - --caCertFile=/etc/istio/certs/root-cert.pem + - --tlsCertFile=/etc/istio/certs/cert-chain.pem + - --tlsKeyFile=/etc/istio/certs/key.pem + - --injectConfig=/etc/istio/inject/config + - --meshConfig=/etc/istio/config/mesh + - --healthCheckInterval=2s + - --healthCheckFile=/health + volumeMounts: + - name: config-volume + mountPath: /etc/istio/config + readOnly: true + - name: certs + mountPath: /etc/istio/certs + readOnly: true + - name: inject-config + mountPath: /etc/istio/inject + readOnly: true + livenessProbe: + exec: + command: + - /usr/local/bin/sidecar-injector + - probe + - --probe-path=/health + - --interval=4s + initialDelaySeconds: 4 + periodSeconds: 4 + readinessProbe: + exec: + command: + - /usr/local/bin/sidecar-injector + - probe + - --probe-path=/health + - --interval=4s + initialDelaySeconds: 4 + periodSeconds: 4 + resources: +{{- if .Values.resources }} +{{ toYaml .Values.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + volumes: + - name: config-volume + configMap: + name: istio + - name: certs + secret: + secretName: istio.istio-sidecar-injector-service-account + - name: inject-config + configMap: + name: istio-sidecar-injector + items: + - key: config + path: config + - key: values + path: values + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/mutatingwebhook.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/mutatingwebhook.yaml new file mode 100755 index 000000000..a30dd38e5 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/mutatingwebhook.yaml @@ -0,0 +1,39 @@ +apiVersion: admissionregistration.k8s.io/v1beta1 +kind: MutatingWebhookConfiguration +metadata: + name: istio-sidecar-injector + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +webhooks: + - name: sidecar-injector.istio.io + clientConfig: + service: + name: istio-sidecar-injector + namespace: {{ .Release.Namespace }} + path: "/inject" + caBundle: "" + rules: + - operations: [ "CREATE" ] + apiGroups: [""] + apiVersions: ["v1"] + resources: ["pods"] + failurePolicy: Fail + namespaceSelector: +{{- if .Values.enableNamespacesByDefault }} + matchExpressions: + - key: name + operator: NotIn + values: + - {{ .Release.Namespace }} + - key: istio-injection + operator: NotIn + values: + - disabled +{{- else }} + matchLabels: + istio-injection: enabled +{{- end }} + diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..51fb3fc3e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/poddisruptionbudget.yaml @@ -0,0 +1,18 @@ +{{- if .Values.global.defaultPodDisruptionBudget.enabled }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: istio-sidecar-injector + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + release: {{ .Release.Name }} + istio: sidecar-injector +spec: +{{ include "podDisruptionBudget.spec" .Values.global.defaultPodDisruptionBudget }} + selector: + matchLabels: + app: {{ template "sidecar-injector.name" . }} + release: {{ .Release.Name }} + istio: sidecar-injector + {{- end }} \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/service.yaml new file mode 100755 index 000000000..537e6f960 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-sidecar-injector + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector +spec: + ports: + - port: 443 + name: https-inject + - port: {{ .Values.global.monitoringPort }} + name: http-monitoring + selector: + istio: sidecar-injector diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/serviceaccount.yaml new file mode 100755 index 000000000..d4020b517 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} +metadata: + name: istio-sidecar-injector-service-account + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "sidecar-injector.name" . }} + chart: {{ template "sidecar-injector.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector diff --git a/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/values.yaml new file mode 100755 index 000000000..31980ac19 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/sidecarInjectorWebhook/values.yaml @@ -0,0 +1,44 @@ +# +# sidecar-injector webhook configuration +# +enabled: true +replicaCount: 1 +rollingMaxSurge: 100% +rollingMaxUnavailable: 25% +image: sidecar_injector +enableNamespacesByDefault: false +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +# If true, webhook or istioctl injector will rewrite PodSpec for liveness +# health check to redirect request to sidecar. This makes liveness check work +# even when mTLS is enabled. +rewriteAppHTTPProbe: false + +# You can use the field called alwaysInjectSelector and neverInjectSelector which will always inject the sidecar or +# always skip the injection on pods that match that label selector, regardless of the global policy. +# See https://istio.io/docs/setup/kubernetes/additional-setup/sidecar-injection/#more-control-adding-exceptions +neverInjectSelector: [] + +alwaysInjectSelector: [] \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/Chart.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/Chart.yaml new file mode 100755 index 000000000..4f4cc100d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +appVersion: 1.5.1 +description: A Helm chart for Kubernetes +name: tracing +tillerVersion: '>=2.7.2' +version: 1.3.3 diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/_helpers.tpl new file mode 100755 index 000000000..e246b59b1 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "tracing.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "tracing.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "tracing.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-jaeger.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-jaeger.yaml new file mode 100755 index 000000000..cc7f8999e --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-jaeger.yaml @@ -0,0 +1,117 @@ +{{ if eq .Values.provider "jaeger" }} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-tracing + namespace: {{ .Release.Namespace }} + labels: + app: jaeger + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + matchLabels: + app: jaeger + template: + metadata: + labels: + app: jaeger + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + prometheus.io/scrape: "true" + prometheus.io/port: "16686" +{{- if .Values.contextPath }} + prometheus.io/path: "{{ .Values.contextPath }}/metrics" +{{- else }} + prometheus.io/path: "/{{ .Values.provider }}/metrics" +{{- end }} + spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} + containers: + - name: jaeger + image: "{{ .Values.jaeger.hub }}/{{ .Values.jaeger.image }}:{{ .Values.jaeger.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: 9411 + - containerPort: 16686 + - containerPort: 5775 + protocol: UDP + - containerPort: 6831 + protocol: UDP + - containerPort: 6832 + protocol: UDP + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + {{- if eq .Values.jaeger.spanStorageType "badger" }} + - name: BADGER_EPHEMERAL + value: "false" + - name: SPAN_STORAGE_TYPE + value: "badger" + - name: BADGER_DIRECTORY_VALUE + value: "/badger/data" + - name: BADGER_DIRECTORY_KEY + value: "/badger/key" + {{- end }} + - name: COLLECTOR_ZIPKIN_HTTP_PORT + value: "9411" + - name: MEMORY_MAX_TRACES + value: "{{ .Values.jaeger.memory.max_traces }}" + - name: QUERY_BASE_PATH + value: {{ if .Values.contextPath }} {{ .Values.contextPath }} {{ else }} /{{ .Values.provider }} {{ end }} + livenessProbe: + httpGet: + path: / + port: 16686 + readinessProbe: + httpGet: + path: / + port: 16686 +{{- if eq .Values.jaeger.spanStorageType "badger" }} + volumeMounts: + - name: data + mountPath: /badger +{{- end }} + resources: +{{- if .Values.jaeger.resources }} +{{ toYaml .Values.jaeger.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} +{{- if eq .Values.jaeger.spanStorageType "badger" }} + volumes: + - name: data +{{- if .Values.jaeger.persist }} + persistentVolumeClaim: + claimName: istio-jaeger-pvc +{{- else }} + emptyDir: {} +{{- end }} +{{- end }} +{{ end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-zipkin.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-zipkin.yaml new file mode 100755 index 000000000..04dfbb033 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/deployment-zipkin.yaml @@ -0,0 +1,82 @@ +{{ if eq .Values.provider "zipkin" }} + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: istio-tracing + namespace: {{ .Release.Namespace }} + labels: + app: zipkin + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + selector: + matchLabels: + app: zipkin + template: + metadata: + labels: + app: zipkin + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + sidecar.istio.io/inject: "false" + scheduler.alpha.kubernetes.io/critical-pod: "" + spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} +{{- if .Values.global.imagePullSecrets }} + imagePullSecrets: +{{- range .Values.global.imagePullSecrets }} + - name: {{ . }} +{{- end }} +{{- end }} + containers: + - name: zipkin + image: "{{ .Values.zipkin.hub }}/{{ .Values.zipkin.image }}:{{ .Values.zipkin.tag }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + ports: + - containerPort: {{ .Values.zipkin.queryPort }} + livenessProbe: + initialDelaySeconds: {{ .Values.zipkin.probeStartupDelay }} + tcpSocket: + port: {{ .Values.zipkin.queryPort }} + readinessProbe: + initialDelaySeconds: {{ .Values.zipkin.probeStartupDelay }} + httpGet: + path: /health + port: {{ .Values.zipkin.queryPort }} + resources: +{{- if .Values.zipkin.resources }} +{{ toYaml .Values.zipkin.resources | indent 12 }} +{{- else }} +{{ toYaml .Values.global.defaultResources | indent 12 }} +{{- end }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: QUERY_PORT + value: "{{ .Values.zipkin.queryPort }}" + - name: JAVA_OPTS + value: "-XX:ConcGCThreads={{ .Values.zipkin.node.cpus }} -XX:ParallelGCThreads={{ .Values.zipkin.node.cpus }} -Djava.util.concurrent.ForkJoinPool.common.parallelism={{ .Values.zipkin.node.cpus }} -Xms{{ .Values.zipkin.javaOptsHeap }}M -Xmx{{ .Values.zipkin.javaOptsHeap }}M -XX:+UseG1GC -server" + - name: STORAGE_METHOD + value: "mem" + - name: ZIPKIN_STORAGE_MEM_MAXSPANS + value: "{{ .Values.zipkin.maxSpans }}" + affinity: + {{- include "nodeaffinity" . | indent 6 }} + {{- include "podAntiAffinity" . | indent 6 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 6 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 6 }} + {{- end }} +{{ end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/ingress.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/ingress.yaml new file mode 100755 index 000000000..72f362166 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/ingress.yaml @@ -0,0 +1,41 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: {{ template "tracing.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.provider }} + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + annotations: + {{- range $key, $value := .Values.ingress.annotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} +spec: + rules: +{{- if .Values.ingress.hosts }} + {{- range $host := .Values.ingress.hosts }} + - host: {{ $host }} + http: + paths: + - path: {{ if $.Values.contextPath }} {{ $.Values.contextPath }} {{ else }} /{{ $.Values.provider }} {{ end }} + backend: + serviceName: tracing + servicePort: 80 + + {{- end -}} +{{- else }} + - http: + paths: + - path: {{ if .Values.contextPath }} {{ .Values.contextPath }} {{ else }} /{{ .Values.provider }} {{ end }} + backend: + serviceName: tracing + servicePort: 80 +{{- end }} + {{- if .Values.ingress.tls }} + tls: +{{ toYaml .Values.ingress.tls | indent 4 }} + {{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/pvc.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/pvc.yaml new file mode 100755 index 000000000..60140494c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/pvc.yaml @@ -0,0 +1,21 @@ +{{- if eq .Values.provider "jaeger" }} +{{- if .Values.jaeger.persist }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: istio-jaeger-pvc + namespace: {{ .Release.Namespace }} + labels: + app: jaeger + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +spec: + storageClassName: {{ .Values.jaeger.storageClassName }} + accessModes: + - {{ .Values.jaeger.accessMode }} + resources: + requests: + storage: 5Gi +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service-jaeger.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service-jaeger.yaml new file mode 100755 index 000000000..23979baf8 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service-jaeger.yaml @@ -0,0 +1,90 @@ +{{ if eq .Values.provider "jaeger" }} + +apiVersion: v1 +kind: List +metadata: + name: jaeger-services + namespace: {{ .Release.Namespace }} + labels: + app: jaeger + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +items: +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-query + namespace: {{ .Release.Namespace }} + annotations: + {{- range $key, $val := .Values.service.annotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + labels: + app: jaeger + jaeger-infra: jaeger-service + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + ports: + - name: query-http + port: 16686 + protocol: TCP + targetPort: 16686 + selector: + app: jaeger +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-collector + namespace: {{ .Release.Namespace }} + labels: + app: jaeger + jaeger-infra: collector-service + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + ports: + - name: jaeger-collector-tchannel + port: 14267 + protocol: TCP + targetPort: 14267 + - name: jaeger-collector-http + port: 14268 + targetPort: 14268 + protocol: TCP + selector: + app: jaeger + type: ClusterIP +- apiVersion: v1 + kind: Service + metadata: + name: jaeger-agent + namespace: {{ .Release.Namespace }} + labels: + app: jaeger + jaeger-infra: agent-service + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + ports: + - name: agent-zipkin-thrift + port: 5775 + protocol: UDP + targetPort: 5775 + - name: agent-compact + port: 6831 + protocol: UDP + targetPort: 6831 + - name: agent-binary + port: 6832 + protocol: UDP + targetPort: 6832 + clusterIP: None + selector: + app: jaeger +{{ end }} + diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service.yaml new file mode 100755 index 000000000..fe94067b0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/service.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: List +metadata: + name: tracing-services + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.provider }} + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +items: +- apiVersion: v1 + kind: Service + metadata: + name: zipkin + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.provider }} + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.externalPort }} + targetPort: 9411 + protocol: TCP + name: {{ .Values.service.name }} + selector: + app: {{ .Values.provider }} +- apiVersion: v1 + kind: Service + metadata: + name: tracing + namespace: {{ .Release.Namespace }} + annotations: + {{- range $key, $val := .Values.service.annotations }} + {{ $key }}: {{ $val | quote }} + {{- end }} + labels: + app: {{ .Values.provider }} + chart: {{ template "tracing.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + spec: + ports: + - name: http-query + port: 80 + protocol: TCP +{{ if eq .Values.provider "jaeger" }} + targetPort: 16686 +{{ else }} + targetPort: 9411 +{{ end}} + selector: + app: {{ .Values.provider }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/tests/test-tracing-connection.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/tests/test-tracing-connection.yaml new file mode 100755 index 000000000..b87f487e0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/templates/tests/test-tracing-connection.yaml @@ -0,0 +1,40 @@ +{{- if .Values.global.enableHelmTest }} +apiVersion: v1 +kind: Pod +metadata: + name: {{ .Release.Name }}-{{ .Values.provider }}-test + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.provider }}-test + chart: {{ template "tracing.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + sidecar.istio.io/inject: "false" + helm.sh/hook: test-success +spec: +{{- if .Values.global.priorityClassName }} + priorityClassName: "{{ .Values.global.priorityClassName }}" +{{- end }} + containers: + - name: "{{ .Values.provider }}-test" + image: pstauffer/curl:v1.0.3 + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" + command: ['curl'] + {{- if eq .Values.provider "jaeger" }} + args: ['http://tracing:80{{ .Values.jaeger.contextPath}}'] + {{- else }} + args: ['http://tracing:80'] + {{- end }} + restartPolicy: Never + affinity: + {{- include "nodeaffinity" . | indent 4 }} + {{- include "podAntiAffinity" . | indent 4 }} + {{- if .Values.tolerations }} + tolerations: +{{ toYaml .Values.tolerations | indent 2 }} + {{- else if .Values.global.defaultTolerations }} + tolerations: +{{ toYaml .Values.global.defaultTolerations | indent 2 }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/charts/tracing/values.yaml b/kubernetes/helm_charts/istio-system/istio/charts/tracing/values.yaml new file mode 100755 index 000000000..f0d386f6f --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/charts/tracing/values.yaml @@ -0,0 +1,84 @@ +# +# addon jaeger tracing configuration +# +enabled: false + +provider: jaeger +nodeSelector: {} +tolerations: [] + +# Specify the pod anti-affinity that allows you to constrain which nodes +# your pod is eligible to be scheduled based on labels on pods that are +# already running on the node rather than based on labels on nodes. +# There are currently two types of anti-affinity: +# "requiredDuringSchedulingIgnoredDuringExecution" +# "preferredDuringSchedulingIgnoredDuringExecution" +# which denote "hard" vs. "soft" requirements, you can define your values +# in "podAntiAffinityLabelSelector" and "podAntiAffinityTermLabelSelector" +# correspondingly. +# For example: +# podAntiAffinityLabelSelector: +# - key: security +# operator: In +# values: S1,S2 +# topologyKey: "kubernetes.io/hostname" +# This pod anti-affinity rule says that the pod requires not to be scheduled +# onto a node if that node is already running a pod with label having key +# "security" and value "S1". +podAntiAffinityLabelSelector: [] +podAntiAffinityTermLabelSelector: [] + +jaeger: + hub: docker.io/jaegertracing + image: all-in-one + tag: 1.12 + memory: + max_traces: 50000 + # spanStorageType value can be "memory" and "badger" for all-in-one image + spanStorageType: badger + persist: false + storageClassName: "" + accessMode: ReadWriteMany + +zipkin: + hub: docker.io/openzipkin + image: zipkin + tag: 2.14.2 + probeStartupDelay: 200 + queryPort: 9411 + resources: + limits: + cpu: 300m + memory: 900Mi + requests: + cpu: 150m + memory: 900Mi + javaOptsHeap: 700 + # From: https://github.com/openzipkin/zipkin/blob/master/zipkin-server/src/main/resources/zipkin-server-shared.yml#L51 + # Maximum number of spans to keep in memory. When exceeded, oldest traces (and their spans) will be purged. + # A safe estimate is 1K of memory per span (each span with 2 annotations + 1 binary annotation), plus + # 100 MB for a safety buffer. You'll need to verify in your own environment. + maxSpans: 500000 + node: + cpus: 2 + +service: + annotations: {} + name: http + type: ClusterIP + externalPort: 9411 + +ingress: + enabled: false + # Used to create an Ingress record. + hosts: + # - tracing.local + annotations: + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + tls: + # Secrets must be manually created in the namespace. + # - secretName: tracing-tls + # hosts: + # - tracing.local + diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/README.md b/kubernetes/helm_charts/istio-system/istio/example-values/README.md new file mode 100755 index 000000000..74fedcb60 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/README.md @@ -0,0 +1,5 @@ +# Example Values + +These files provide various example values for different Istio setups. + +To use them, [read the docs](https://istio.io/docs/setup/kubernetes/helm-install/) and add the flag `--values example-file.yaml`. diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-example-sds-vault.yaml b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-example-sds-vault.yaml new file mode 100755 index 000000000..1016afc01 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-example-sds-vault.yaml @@ -0,0 +1,26 @@ +global: + controlPlaneSecurityEnabled: false + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: true + + sds: + enabled: true + udsPath: "unix:/var/run/sds/uds_path" + +nodeagent: + enabled: true + image: node-agent-k8s + env: + # The IP address and the port number of a publicly accessible example Vault server. + CA_ADDR: "https://34.83.129.211:8200" + CA_PROVIDER: "VaultCA" + VALID_TOKEN: true + # The IP address and the port number of a publicly accessible example Vault server. + VAULT_ADDR: "https://34.83.129.211:8200" + VAULT_AUTH_PATH: "auth/kubernetes/login" + VAULT_ROLE: "istio-cert" + VAULT_SIGN_CSR_PATH: "istio_ca/sign/istio-pki-role" + VAULT_TLS_ROOT_CERT: '-----BEGIN CERTIFICATE-----\nMIIC3jCCAcagAwIBAgIRAO1S7vuRQmo2He+RtBq3fv8wDQYJKoZIhvcNAQELBQAw\nEDEOMAwGA1UEChMFVmF1bHQwIBcNMTkwNDI3MTY1ODE1WhgPMjExOTA0MDMxNjU4\nMTVaMBAxDjAMBgNVBAoTBVZhdWx0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB\nCgKCAQEA7/CTbnENEIvFZg9hmVtYnOx3OfMy/GNCuP7sqtAeVVTopAKKkcAAWQck\nrhpBooEGpCugNxXGNCuJh/2nu0AfGFRfdafwSJRoI6yHwQouDm0o4r3h9uL3tu5N\nD+x9j+eejbFsoZVn84CxGkEB6oyeXYHjc6eWh3PFGMtKuOQD4pezvDH0yNCx5waK\nhtPuYtl0ebfdbyh+WQuptO+Q9VSaQNqE3ipZ461y8PduwRRll241W0gQB2iasX03\nD36F2ZrMz3KEVRVKM1yCUDCy2RPJqkXPdnVMWmDGbe8Uw69zr25JltzuRZFT9HL3\nY1RnMTecmSc4ikTUHcMhFX3PYbfR5wIDAQABozEwLzAOBgNVHQ8BAf8EBAMCBaAw\nDAYDVR0TAQH/BAIwADAPBgNVHREECDAGhwQiU4HTMA0GCSqGSIb3DQEBCwUAA4IB\nAQCdLh6olDVQB71LD6srbfAE4EsxLEBbIRnv7Nf1S0KQwgW/QxK8DHBwJBxJkr1N\nzgEPx86f2Fo2UsY9m6rvgP3+iquyMsKi0ooUah3y3LSnONuZcdfSTl/HYd38S6Dp\nVkVOZ7781xxpFVUqQ5voQX1Y1Ipn5qw0FyIcNYWLkNX+iMf1b9kpEIWQNhRC/Yiv\nTS0VA/BzQemGyf2UB6QsuZLH+JFEZnzU859qURnNIITa1Wf4YUtka5Sp1kDnEll3\nwj4IlXKU+Wl1CzxJyn4SSQAXy/Lb08ZKrF/YSzcIISnRX5j+wa8ApOSwwA/B7iaT\nTWz1g+RlV9qHap70eIjPsQvb\n-----END CERTIFICATE-----' \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-gateways.yaml b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-gateways.yaml new file mode 100755 index 000000000..b9930d0a0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-gateways.yaml @@ -0,0 +1,135 @@ +# Common settings. +global: + # Omit the istio-sidecar-injector configmap when generate a + # standalone gateway. Gateways may be created in namespaces other + # than `istio-system` and we don't want to re-create the injector + # configmap in those. + omitSidecarInjectorConfigMap: true + + # Istio control plane namespace: This specifies where the Istio control + # plane was installed earlier. Modify this if you installed the control + # plane in a different namespace than istio-system. + istioNamespace: istio-system + + proxy: + # Sets the destination Statsd in envoy (the value of the "--statsdUdpAddress" proxy argument + # would be :). + # Disabled by default. + # The istio-statsd-prom-bridge is deprecated and should not be used moving forward. + envoyStatsd: + # If enabled is set to true, host and port must also be provided. Istio no longer provides a statsd collector. + enabled: false + host: # example: statsd-svc.istio-system + port: # example: 9125 + + +# +# Gateways Configuration +# By default (if enabled) a pair of Ingress and Egress Gateways will be created for the mesh. +# You can add more gateways in addition to the defaults but make sure those are uniquely named +# and that NodePorts are not conflicting. +# Disable specific gateway by setting the `enabled` to false. +# +gateways: + enabled: true + + custom-gateway: + enabled: true + labels: + app: custom-gateway + replicaCount: 1 + autoscaleMin: 1 + autoscaleMax: 5 + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + #requests: + # cpu: 1800m + # memory: 256Mi + cpu: + targetAverageUtilization: 80 + loadBalancerIP: "" + loadBalancerSourceRanges: {} + externalIPs: [] + serviceAnnotations: {} + podAnnotations: {} + type: LoadBalancer #change to NodePort, ClusterIP or LoadBalancer if need be + #externalTrafficPolicy: Local #change to Local to preserve source IP or Cluster for default behaviour or leave commented out + ports: + ## You can add custom gateway ports + - port: 80 + targetPort: 80 + name: http2 + # nodePort: 31380 + - port: 443 + name: https + # nodePort: 31390 + - port: 31400 + name: tcp + # nodePort: 31400 + # Pilot and Citadel MTLS ports are enabled in gateway - but will only redirect + # to pilot/citadel if global.meshExpansion settings are enabled. + - port: 15011 + targetPort: 15011 + name: tcp-pilot-grpc-tls + - port: 8060 + targetPort: 8060 + name: tcp-citadel-grpc-tls + # Addon ports for kiali are enabled in gateway - but will only redirect if + # the gateway configuration for the various components are enabled. + - port: 15029 + targetPort: 15029 + name: http2-kiali + # Telemetry-related ports are enabled in gateway - but will only redirect if + # the gateway configuration for the various components are enabled. + - port: 15030 + targetPort: 15030 + name: http2-prometheus + - port: 15031 + targetPort: 15031 + name: http2-grafana + - port: 15032 + targetPort: 15032 + name: http2-tracing + secretVolumes: + - name: customgateway-certs + secretName: istio-customgateway-certs + mountPath: /etc/istio/customgateway-certs + - name: customgateway-ca-certs + secretName: istio-customgateway-ca-certs + mountPath: /etc/istio/customgateway-ca-certs + +# all other components are disabled except the gateways +security: + enabled: false + +sidecarInjectorWebhook: + enabled: false + +galley: + enabled: false + +mixer: + policy: + enabled: false + telemetry: + enabled: false + +pilot: + enabled: false + +grafana: + enabled: false + +prometheus: + enabled: false + +tracing: + enabled: false + +kiali: + enabled: false + +certmanager: + enabled: false diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-googleca.yaml b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-googleca.yaml new file mode 100755 index 000000000..0e6d15342 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-googleca.yaml @@ -0,0 +1,28 @@ +global: + controlPlaneSecurityEnabled: true + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: true + + sds: + enabled: true + udsPath: "unix:/var/run/sds/uds_path" + # For GoogleCA, the aud field need to be set to the trustDomain, which is also set at the + # installation/configuration time, e.g. by running helm template. + token: + aud: "" + + trustDomain: "" + + useMCP: true + +nodeagent: + enabled: true + image: node-agent-k8s + env: + CA_PROVIDER: "GoogleCA" + CA_ADDR: "meshca.googleapis.com:443" + PLUGINS: "GoogleTokenExchange" + GKE_CLUSTER_URL: "" diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-meshexpansion-gateways.yaml b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-meshexpansion-gateways.yaml new file mode 100755 index 000000000..51a1ded16 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-meshexpansion-gateways.yaml @@ -0,0 +1,28 @@ +global: + # Provides dns resolution for global services + podDNSSearchNamespaces: + - global + + multiCluster: + enabled: true + + controlPlaneSecurityEnabled: true + meshExpansion: + enabled: true + +# Multicluster with gateways requires a root CA +# Cluster local CAs are bootstrapped with the root CA. +security: + selfSigned: false + +# Provides dns resolution for service entries of form +# name.namespace.global +istiocoredns: + enabled: true + +gateways: + istio-egressgateway: + enabled: true + env: + # Needed to route traffic via egress gateway if desired. + ISTIO_META_REQUESTED_NETWORK_VIEW: "external" diff --git a/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-multicluster-gateways.yaml b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-multicluster-gateways.yaml new file mode 100755 index 000000000..3524a3d47 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/example-values/values-istio-multicluster-gateways.yaml @@ -0,0 +1,27 @@ +global: + # Provides dns resolution for global services + podDNSSearchNamespaces: + - global + - "{{ valueOrDefault .DeploymentMeta.Namespace \"default\" }}.global" + + multiCluster: + enabled: true + + controlPlaneSecurityEnabled: true + +# Multicluster with gateways requires a root CA +# Cluster local CAs are bootstrapped with the root CA. +security: + selfSigned: false + +# Provides dns resolution for service entries of form +# name.namespace.global +istiocoredns: + enabled: true + +gateways: + istio-egressgateway: + enabled: true + env: + # Needed to route traffic via egress gateway if desired. + ISTIO_META_REQUESTED_NETWORK_VIEW: "external" diff --git a/kubernetes/helm_charts/istio-system/istio/files/injection-template.yaml b/kubernetes/helm_charts/istio-system/istio/files/injection-template.yaml new file mode 100755 index 000000000..49395a181 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/files/injection-template.yaml @@ -0,0 +1,388 @@ +rewriteAppHTTPProbe: {{ valueOrDefault .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe false }} +{{- if or (not .Values.istio_cni.enabled) .Values.global.proxy.enableCoreDump }} +initContainers: +{{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} +{{- if not .Values.istio_cni.enabled }} +- name: istio-init +{{- if contains "/" .Values.global.proxy_init.image }} + image: "{{ .Values.global.proxy_init.image }}" +{{- else }} + image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" +{{- end }} + args: + - "-p" + - "15001" + - "-z" + - "15006" + - "-u" + - 1337 + - "-m" + - "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" + - "-i" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" + - "-x" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" + - "-b" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}" + - "-d" + - "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" + {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") -}} + - "-o" + - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" + {{ end -}} + {{ if (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces`) -}} + - "-k" + - "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" + {{ end -}} + imagePullPolicy: "{{ .Values.global.imagePullPolicy }}" +{{- if .Values.global.proxy.init.resources }} + resources: + {{ toYaml .Values.global.proxy.init.resources | indent 4 }} +{{- else }} + resources: {} +{{- end }} + securityContext: + runAsUser: 0 + runAsNonRoot: false + capabilities: + add: + - NET_ADMIN + {{- if .Values.global.proxy.privileged }} + privileged: true + {{- end }} + restartPolicy: Always +{{- end }} +{{ end -}} +{{- if eq .Values.global.proxy.enableCoreDump true }} +- name: enable-core-dump + args: + - -c + - sysctl -w kernel.core_pattern=/var/lib/istio/core.proxy && ulimit -c unlimited + command: + - /bin/sh + image: {{ $.Values.global.proxy.enableCoreDumpImage }} + imagePullPolicy: IfNotPresent + resources: {} + securityContext: + runAsUser: 0 + runAsNonRoot: false + privileged: true +{{ end }} +{{- end }} +containers: +- name: istio-proxy +{{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image) }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}" +{{- else }} + image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}" +{{- end }} + ports: + - containerPort: 15090 + protocol: TCP + name: http-envoy-prom + args: + - proxy + - sidecar + - --domain + - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} + - --configPath + - "{{ .ProxyConfig.ConfigPath }}" + - --binaryPath + - "{{ .ProxyConfig.BinaryPath }}" + - --serviceCluster + {{ if ne "" (index .ObjectMeta.Labels "app") -}} + - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)" + {{ else -}} + - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}" + {{ end -}} + - --drainDuration + - "{{ formatDuration .ProxyConfig.DrainDuration }}" + - --parentShutdownDuration + - "{{ formatDuration .ProxyConfig.ParentShutdownDuration }}" + - --discoveryAddress + - "{{ annotation .ObjectMeta `sidecar.istio.io/discoveryAddress` .ProxyConfig.DiscoveryAddress }}" +{{- if eq .Values.global.proxy.tracer "lightstep" }} + - --lightstepAddress + - "{{ .ProxyConfig.GetTracing.GetLightstep.GetAddress }}" + - --lightstepAccessToken + - "{{ .ProxyConfig.GetTracing.GetLightstep.GetAccessToken }}" + - --lightstepSecure={{ .ProxyConfig.GetTracing.GetLightstep.GetSecure }} + - --lightstepCacertPath + - "{{ .ProxyConfig.GetTracing.GetLightstep.GetCacertPath }}" +{{- else if eq .Values.global.proxy.tracer "zipkin" }} + - --zipkinAddress + - "{{ .ProxyConfig.GetTracing.GetZipkin.GetAddress }}" +{{- else if eq .Values.global.proxy.tracer "datadog" }} + - --datadogAgentAddress + - "{{ .ProxyConfig.GetTracing.GetDatadog.GetAddress }}" +{{- end }} +{{- if .Values.global.proxy.logLevel }} + - --proxyLogLevel={{ .Values.global.proxy.logLevel }} +{{- end}} +{{- if .Values.global.proxy.componentLogLevel }} + - --proxyComponentLogLevel={{ .Values.global.proxy.componentLogLevel }} +{{- end}} + - --dnsRefreshRate + - {{ .Values.global.proxy.dnsRefreshRate }} + - --connectTimeout + - "{{ formatDuration .ProxyConfig.ConnectTimeout }}" +{{- if .Values.global.proxy.envoyStatsd.enabled }} + - --statsdUdpAddress + - "{{ .ProxyConfig.StatsdUdpAddress }}" +{{- end }} +{{- if .Values.global.proxy.envoyMetricsService.enabled }} + - --envoyMetricsServiceAddress + - "{{ .ProxyConfig.GetEnvoyMetricsService.GetAddress }}" +{{- end }} +{{- if .Values.global.proxy.envoyAccessLogService.enabled }} + - --envoyAccessLogService + - '{{ structToJSON .ProxyConfig.EnvoyAccessLogService }}' +{{- end }} + - --proxyAdminPort + - "{{ .ProxyConfig.ProxyAdminPort }}" + {{ if gt .ProxyConfig.Concurrency 0 -}} + - --concurrency + - "{{ .ProxyConfig.Concurrency }}" + {{ end -}} + - --controlPlaneAuthPolicy + - "{{ annotation .ObjectMeta `sidecar.istio.io/controlPlaneAuthPolicy` .ProxyConfig.ControlPlaneAuthPolicy }}" +{{- if (ne (annotation .ObjectMeta "status.sidecar.istio.io/port" .Values.global.proxy.statusPort) "0") }} + - --statusPort + - "{{ annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort }}" + - --applicationPorts + - "{{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/applicationPorts` (applicationPorts .Spec.Containers) }}" +{{- end }} +{{- if .Values.global.trustDomain }} + - --trust-domain={{ .Values.global.trustDomain }} +{{- end }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ISTIO_META_POD_PORTS + value: |- + [ + {{- range $index1, $c := .Spec.Containers }} + {{- range $index2, $p := $c.Ports }} + {{if or (ne $index1 0) (ne $index2 0)}},{{end}}{{ structToJSON $p }} + {{- end}} + {{- end}} + ] + - name: ISTIO_META_CLUSTER_ID + value: "{{ valueOrDefault .Values.global.multicluster.clusterName `Kubernetes` }}" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName +{{- if eq .Values.global.proxy.tracer "datadog" }} + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +{{- if isset .ObjectMeta.Annotations `apm.datadoghq.com/env` }} +{{- range $key, $value := fromJSON (index .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} + - name: {{ $key }} + value: "{{ $value }}" +{{- end }} +{{- end }} +{{- end }} + - name: ISTIO_META_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ISTIO_META_CONFIG_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SDS_ENABLED + value: {{ $.Values.global.sds.enabled }} + - name: ISTIO_META_INTERCEPTION_MODE + value: "{{ or (index .ObjectMeta.Annotations `sidecar.istio.io/interceptionMode`) .ProxyConfig.InterceptionMode.String }}" + - name: ISTIO_META_INCLUDE_INBOUND_PORTS + value: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` (applicationPorts .Spec.Containers) }}" + {{- if .Values.global.network }} + - name: ISTIO_META_NETWORK + value: "{{ .Values.global.network }}" + {{- end }} + {{ if .ObjectMeta.Annotations }} + - name: ISTIO_METAJSON_ANNOTATIONS + value: | + {{ toJSON .ObjectMeta.Annotations }} + {{ end }} + {{ if .ObjectMeta.Labels }} + - name: ISTIO_METAJSON_LABELS + value: | + {{ toJSON .ObjectMeta.Labels }} + {{ end }} + {{- if .DeploymentMeta.Name }} + - name: ISTIO_META_WORKLOAD_NAME + value: {{ .DeploymentMeta.Name }} + {{ end }} + {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }} + - name: ISTIO_META_OWNER + value: kubernetes://api/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }} + {{- end}} + {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} + - name: ISTIO_BOOTSTRAP_OVERRIDE + value: "/etc/istio/custom-bootstrap/custom_bootstrap.json" + {{- end }} + {{- if .Values.global.sds.customTokenDirectory }} + - name: ISTIO_META_SDS_TOKEN_PATH + value: "{{ .Values.global.sds.customTokenDirectory -}}/sdstoken" + {{- end }} + {{- if .Values.global.meshID }} + - name: ISTIO_META_MESH_ID + value: "{{ .Values.global.meshID }}" + {{- else if .Values.global.trustDomain }} + - name: ISTIO_META_MESH_ID + value: "{{ .Values.global.trustDomain }}" + {{- end }} + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} + readinessProbe: + httpGet: + path: /healthz/ready + port: {{ annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort }} + initialDelaySeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/initialDelaySeconds` .Values.global.proxy.readinessInitialDelaySeconds }} + periodSeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/periodSeconds` .Values.global.proxy.readinessPeriodSeconds }} + failureThreshold: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/failureThreshold` .Values.global.proxy.readinessFailureThreshold }} + {{ end -}} + securityContext: + {{- if .Values.global.proxy.privileged }} + privileged: true + {{- end }} + {{- if ne .Values.global.proxy.enableCoreDump true }} + readOnlyRootFilesystem: true + {{- end }} + {{ if eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY` -}} + capabilities: + add: + - NET_ADMIN + runAsGroup: 1337 + {{ else -}} + {{ if .Values.global.sds.enabled }} + runAsGroup: 1337 + {{- end }} + runAsUser: 1337 + {{- end }} + resources: + {{ if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} + requests: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} + cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" + {{ end}} + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} + memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" + {{ end }} + {{ else -}} +{{- if .Values.global.proxy.resources }} + {{ toYaml .Values.global.proxy.resources | indent 4 }} +{{- end }} + {{ end -}} + volumeMounts: + {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} + - mountPath: /etc/istio/custom-bootstrap + name: custom-bootstrap-volume + {{- end }} + - mountPath: /etc/istio/proxy + name: istio-envoy + {{- if .Values.global.sds.enabled }} + - mountPath: /var/run/sds + name: sds-uds-path + readOnly: true + - mountPath: /var/run/secrets/tokens + name: istio-token + {{- if .Values.global.sds.customTokenDirectory }} + - mountPath: "{{ .Values.global.sds.customTokenDirectory -}}" + name: custom-sds-token + readOnly: true + {{- end }} + {{- else }} + - mountPath: /etc/certs/ + name: istio-certs + readOnly: true + {{- end }} + {{- if and (eq .Values.global.proxy.tracer "lightstep") .Values.global.tracer.lightstep.cacertPath }} + - mountPath: {{ directory .ProxyConfig.GetTracing.GetLightstep.GetCacertPath }} + name: lightstep-certs + readOnly: true + {{- end }} + {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }} + {{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }} + - name: "{{ $index }}" + {{ toYaml $value | indent 4 }} + {{ end }} + {{- end }} +volumes: +{{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} +- name: custom-bootstrap-volume + configMap: + name: {{ annotation .ObjectMeta `sidecar.istio.io/bootstrapOverride` "" }} +{{- end }} +- emptyDir: + medium: Memory + name: istio-envoy +{{- if .Values.global.sds.enabled }} +- name: sds-uds-path + hostPath: + path: /var/run/sds +- name: istio-token + projected: + sources: + - serviceAccountToken: + path: istio-token + expirationSeconds: 43200 + audience: {{ .Values.global.sds.token.aud }} +{{- if .Values.global.sds.customTokenDirectory }} +- name: custom-sds-token + secret: + secretName: sdstokensecret +{{- end }} +{{- else }} +- name: istio-certs + secret: + optional: true + {{ if eq .Spec.ServiceAccountName "" }} + secretName: istio.default + {{ else -}} + secretName: {{ printf "istio.%s" .Spec.ServiceAccountName }} + {{ end -}} + {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }} + {{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }} +- name: "{{ $index }}" + {{ toYaml $value | indent 2 }} + {{ end }} + {{ end }} +{{- end }} +{{- if and (eq .Values.global.proxy.tracer "lightstep") .Values.global.tracer.lightstep.cacertPath }} +- name: lightstep-certs + secret: + optional: true + secretName: lightstep.cacert +{{- end }} +{{- if .Values.global.podDNSSearchNamespaces }} +dnsConfig: + searches: + {{- range .Values.global.podDNSSearchNamespaces }} + - {{ render . }} + {{- end }} +{{- end }} +podRedirectAnnot: + sidecar.istio.io/interceptionMode: "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" + traffic.sidecar.istio.io/includeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" + traffic.sidecar.istio.io/excludeOutboundIPRanges: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" + traffic.sidecar.istio.io/includeInboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` (includeInboundPorts .Spec.Containers) }}" + traffic.sidecar.istio.io/excludeInboundPorts: "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" +{{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne .Values.global.proxy.excludeOutboundPorts "") }} + traffic.sidecar.istio.io/excludeOutboundPorts: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" +{{- end }} + traffic.sidecar.istio.io/kubevirtInterfaces: "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" diff --git a/kubernetes/helm_charts/istio-system/istio/requirements.yaml b/kubernetes/helm_charts/istio-system/istio/requirements.yaml new file mode 100755 index 000000000..15d238103 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/requirements.yaml @@ -0,0 +1,40 @@ +dependencies: + - name: sidecarInjectorWebhook + version: 1.3.3 + condition: sidecarInjectorWebhook.enabled + - name: security + version: 1.3.3 + condition: security.enabled + - name: gateways + version: 1.3.3 + condition: gateways.enabled + - name: mixer + version: 1.3.3 + condition: or mixer.policy.enabled mixer.telemetry.enabled + - name: nodeagent + version: 1.3.3 + condition: nodeagent.enabled + - name: pilot + version: 1.3.3 + condition: pilot.enabled + - name: grafana + version: 1.3.3 + condition: grafana.enabled + - name: prometheus + version: 1.3.3 + condition: prometheus.enabled + - name: tracing + version: 1.3.3 + condition: tracing.enabled + - name: galley + version: 1.3.3 + condition: galley.enabled + - name: kiali + version: 1.3.3 + condition: kiali.enabled + - name: istiocoredns + version: 1.3.3 + condition: istiocoredns.enabled + - name: certmanager + version: 1.3.3 + condition: certmanager.enabled diff --git a/kubernetes/helm_charts/istio-system/istio/templates/NOTES.txt b/kubernetes/helm_charts/istio-system/istio/templates/NOTES.txt new file mode 100755 index 000000000..3b323594c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/NOTES.txt @@ -0,0 +1,29 @@ +Thank you for installing {{ .Chart.Name | title }}. + +Your release is named {{ .Release.Name | title }}. + +To get started running application with Istio, execute the following steps: + +{{- if index .Values "sidecarInjectorWebhook" "enabled" }} +1. Label namespace that application object will be deployed to by the following command (take default namespace as an example) + +$ kubectl label namespace default istio-injection=enabled +$ kubectl get namespace -L istio-injection + +2. Deploy your applications + +$ kubectl apply -f .yaml +{{- else }} +1. Download the latest release package to get sidecar injection tool + +$ curl -L https://git.io/getLatestIstio | sh - +$ mv istio-* istio-latest +$ export PATH="$PATH:$PWD/istio-latest/bin" + +2. Deploy your application by manually injecting envoy sidecar with `istioctl kube-inject` + +$ kubectl apply -f <(istioctl kube-inject -f .yaml) +{{- end }} + +For more information on running Istio, visit: +https://istio.io/ diff --git a/kubernetes/helm_charts/istio-system/istio/templates/_affinity.tpl b/kubernetes/helm_charts/istio-system/istio/templates/_affinity.tpl new file mode 100755 index 000000000..7639be434 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/_affinity.tpl @@ -0,0 +1,93 @@ +{{/* affinity - https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ */}} + +{{- define "nodeaffinity" }} + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityRequiredDuringScheduling" . }} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "nodeAffinityPreferredDuringScheduling" . }} +{{- end }} + +{{- define "nodeAffinityRequiredDuringScheduling" }} + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + {{- range $key, $val := .Values.global.arch }} + {{- if gt ($val | int) 0 }} + - {{ $key | quote }} + {{- end }} + {{- end }} + {{- $nodeSelector := default .Values.global.defaultNodeSelector .Values.nodeSelector -}} + {{- range $key, $val := $nodeSelector }} + - key: {{ $key }} + operator: In + values: + - {{ $val | quote }} + {{- end }} +{{- end }} + +{{- define "nodeAffinityPreferredDuringScheduling" }} + {{- range $key, $val := .Values.global.arch }} + {{- if gt ($val | int) 0 }} + - weight: {{ $val | int }} + preference: + matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - {{ $key | quote }} + {{- end }} + {{- end }} +{{- end }} + +{{- define "podAntiAffinity" }} +{{- if or .Values.podAntiAffinityLabelSelector .Values.podAntiAffinityTermLabelSelector}} + podAntiAffinity: + {{- if .Values.podAntiAffinityLabelSelector }} + requiredDuringSchedulingIgnoredDuringExecution: + {{- include "podAntiAffinityRequiredDuringScheduling" . }} + {{- end }} + {{- if or .Values.podAntiAffinityTermLabelSelector}} + preferredDuringSchedulingIgnoredDuringExecution: + {{- include "podAntiAffinityPreferredDuringScheduling" . }} + {{- end }} +{{- end }} +{{- end }} + +{{- define "podAntiAffinityRequiredDuringScheduling" }} + {{- range $index, $item := .Values.podAntiAffinityLabelSelector }} + - labelSelector: + matchExpressions: + - key: {{ $item.key }} + operator: {{ $item.operator }} + {{- if $item.values }} + values: + {{- $vals := split "," $item.values }} + {{- range $i, $v := $vals }} + - {{ $v | quote }} + {{- end }} + {{- end }} + topologyKey: {{ $item.topologyKey }} + {{- end }} +{{- end }} + +{{- define "podAntiAffinityPreferredDuringScheduling" }} + {{- range $index, $item := .Values.podAntiAffinityTermLabelSelector }} + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: {{ $item.key }} + operator: {{ $item.operator }} + {{- if $item.values }} + values: + {{- $vals := split "," $item.values }} + {{- range $i, $v := $vals }} + - {{ $v | quote }} + {{- end }} + {{- end }} + topologyKey: {{ $item.topologyKey }} + weight: 100 + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/_helpers.tpl b/kubernetes/helm_charts/istio-system/istio/templates/_helpers.tpl new file mode 100755 index 000000000..f79bea415 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/_helpers.tpl @@ -0,0 +1,46 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "istio.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "istio.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "istio.chart" -}} +{{- .Chart.Name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified configmap name. +*/}} +{{- define "istio.configmap.fullname" -}} +{{- printf "%s-%s" .Release.Name "istio-mesh-config" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Configmap checksum. +*/}} +{{- define "istio.configmap.checksum" -}} +{{- print $.Template.BasePath "/configmap.yaml" | sha256sum -}} +{{- end -}} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/_podDisruptionBudget.tpl b/kubernetes/helm_charts/istio-system/istio/templates/_podDisruptionBudget.tpl new file mode 100755 index 000000000..ebb86068c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/_podDisruptionBudget.tpl @@ -0,0 +1,3 @@ +{{- define "podDisruptionBudget.spec" }} + minAvailable: 1 +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/clusterrole.yaml b/kubernetes/helm_charts/istio-system/istio/templates/clusterrole.yaml new file mode 100755 index 000000000..b92c9ef8b --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/clusterrole.yaml @@ -0,0 +1,11 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: istio-reader +rules: + - apiGroups: [''] + resources: ['nodes', 'pods', 'services', 'endpoints', "replicationcontrollers"] + verbs: ['get', 'watch', 'list'] + - apiGroups: ["extensions", "apps"] + resources: ["replicasets"] + verbs: ["get", "list", "watch"] diff --git a/kubernetes/helm_charts/istio-system/istio/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/istio-system/istio/templates/clusterrolebinding.yaml new file mode 100755 index 000000000..827601b3d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: istio-multi + labels: + chart: {{ .Chart.Name }}-{{ .Chart.Version }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: istio-reader +subjects: +- kind: ServiceAccount + name: istio-multi + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/configmap.yaml b/kubernetes/helm_charts/istio-system/istio/templates/configmap.yaml new file mode 100755 index 000000000..063f2aa76 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/configmap.yaml @@ -0,0 +1,307 @@ +{{- if or .Values.pilot.enabled .Values.global.istioRemote }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istio.name" . }} + chart: {{ template "istio.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} +data: + mesh: |- + # Set the following variable to true to disable policy checks by the Mixer. + # Note that metrics will still be reported to the Mixer. + {{- if .Values.mixer.policy.enabled }} + disablePolicyChecks: {{ .Values.global.disablePolicyChecks }} + {{- else }} + disablePolicyChecks: true + {{- end }} + + {{- if .Values.mixer.telemetry.reportBatchMaxEntries }} + # reportBatchMaxEntries is the number of requests that are batched before telemetry data is sent to the mixer server + reportBatchMaxEntries: {{ .Values.mixer.telemetry.reportBatchMaxEntries }} + {{- end }} + + {{- if .Values.mixer.telemetry.reportBatchMaxTime }} + # reportBatchMaxTime is the max waiting time before the telemetry data of a request is sent to the mixer server + reportBatchMaxTime: {{ .Values.mixer.telemetry.reportBatchMaxTime }} + {{- end }} + + {{- if .Values.mixer.telemetry.sessionAffinityEnabled }} + # sidecarToTelemetrySessionAffinity will create a STRICT_DNS type cluster for istio-telemetry. + sidecarToTelemetrySessionAffinity: {{ .Values.mixer.telemetry.sessionAffinityEnabled }} + {{- end }} + + # Set enableTracing to false to disable request tracing. + enableTracing: {{ .Values.global.enableTracing }} + + # Set accessLogFile to empty string to disable access log. + accessLogFile: "{{ .Values.global.proxy.accessLogFile }}" + + # If accessLogEncoding is TEXT, value will be used directly as the log format + # example: "[%START_TIME%] %REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\n" + # If AccessLogEncoding is JSON, value will be parsed as map[string]string + # example: '{"start_time": "%START_TIME%", "req_method": "%REQ(:METHOD)%"}' + # Leave empty to use default log format + accessLogFormat: {{ .Values.global.proxy.accessLogFormat | quote }} + + # Set accessLogEncoding to JSON or TEXT to configure sidecar access log + accessLogEncoding: '{{ .Values.global.proxy.accessLogEncoding }}' + + enableEnvoyAccessLogService: {{ .Values.global.proxy.envoyAccessLogService.enabled }} + + {{- if .Values.global.istioRemote }} + + {{- if .Values.global.remotePolicyAddress }} + {{- if .Values.global.createRemoteSvcEndpoints }} + mixerCheckServer: istio-policy.{{ .Release.Namespace }}:15004 + {{- else }} + mixerCheckServer: {{ .Values.global.remotePolicyAddress }}:15004 + {{- end }} + {{- end }} + {{- if .Values.global.remoteTelemetryAddress }} + {{- if .Values.global.createRemoteSvcEndpoints }} + mixerReportServer: istio-telemetry.{{ .Release.Namespace }}:15004 + {{- else }} + mixerReportServer: {{ .Values.global.remoteTelemetryAddress }}:15004 + {{- end }} + {{- end }} + + {{- else }} + + {{- if .Values.mixer.policy.enabled }} + {{- if .Values.global.controlPlaneSecurityEnabled }} + mixerCheckServer: istio-policy.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }}:15004 + {{- else }} + mixerCheckServer: istio-policy.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }}:9091 + {{- end }} + {{- end }} + {{- if .Values.mixer.telemetry.enabled }} + {{- if .Values.global.controlPlaneSecurityEnabled }} + mixerReportServer: istio-telemetry.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }}:15004 + {{- else }} + mixerReportServer: istio-telemetry.{{ .Release.Namespace }}.svc.{{ .Values.global.proxy.clusterDomain }}:9091 + {{- end }} + {{- end }} + + {{- end }} + + {{- if or .Values.mixer.policy.enabled (and .Values.global.istioRemote .Values.global.remotePolicyAddress) }} + # policyCheckFailOpen allows traffic in cases when the mixer policy service cannot be reached. + # Default is false which means the traffic is denied when the client is unable to connect to Mixer. + policyCheckFailOpen: {{ .Values.global.policyCheckFailOpen }} + {{- end }} + + {{- if .Values.gateways.enabled }} + # Let Pilot give ingresses the public IP of the Istio ingressgateway + ingressService: istio-ingressgateway + {{- end }} + + # Default connect timeout for dynamic clusters generated by Pilot and returned via XDS + connectTimeout: 10s + + # Automatic protocol detection uses a set of heuristics to + # determine whether the connection is using TLS or not (on the + # server side), as well as the application protocol being used + # (e.g., http vs tcp). These heuristics rely on the client sending + # the first bits of data. For server first protocols like MySQL, + # MongoDB, etc., Envoy will timeout on the protocol detection after + # the specified period, defaulting to non mTLS plain TCP + # traffic. Set this field to tweak the period that Envoy will wait + # for the client to send the first bits of data. (MUST BE >=1ms) + protocolDetectionTimeout: {{ .Values.global.proxy.protocolDetectionTimeout }} + + # DNS refresh rate for Envoy clusters of type STRICT_DNS + dnsRefreshRate: {{ .Values.global.proxy.dnsRefreshRate }} + + # Unix Domain Socket through which envoy communicates with NodeAgent SDS to get + # key/cert for mTLS. Use secret-mount files instead of SDS if set to empty. + sdsUdsPath: {{ .Values.global.sds.udsPath | quote }} + + # The trust domain corresponds to the trust root of a system. + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + trustDomain: {{ .Values.global.trustDomain | quote }} + + # Set the default behavior of the sidecar for handling outbound traffic from the application: + # ALLOW_ANY - outbound traffic to unknown destinations will be allowed, in case there are no + # services or ServiceEntries for the destination port + # REGISTRY_ONLY - restrict outbound traffic to services defined in the service registry as well + # as those defined through ServiceEntries + outboundTrafficPolicy: + mode: {{ .Values.global.outboundTrafficPolicy.mode }} + + {{- if .Values.global.localityLbSetting.enabled }} + localityLbSetting: +{{ toYaml .Values.global.localityLbSetting | trim | indent 6 }} + {{- end }} + # The namespace to treat as the administrative root namespace for istio + # configuration. +{{- if .Values.global.configRootNamespace }} + rootNamespace: {{ .Values.global.configRootNamespace }} +{{- else }} + rootNamespace: {{ .Release.Namespace }} +{{- end }} + + {{- if .Values.global.defaultConfigVisibilitySettings }} + defaultServiceExportTo: + {{- range .Values.global.defaultConfigVisibilitySettings }} + - {{ . | quote }} + {{- end }} + defaultVirtualServiceExportTo: + {{- range .Values.global.defaultConfigVisibilitySettings }} + - {{ . | quote }} + {{- end }} + defaultDestinationRuleExportTo: + {{- range .Values.global.defaultConfigVisibilitySettings }} + - {{ . | quote }} + {{- end }} + {{- end }} + + {{- if $.Values.global.useMCP }} + configSources: + - address: istio-galley.{{ $.Release.Namespace }}.svc:9901 + {{- if $.Values.global.controlPlaneSecurityEnabled}} + tlsSettings: + mode: ISTIO_MUTUAL + {{- end }} + {{- end }} + + defaultConfig: + # + # TCP connection timeout between Envoy & the application, and between Envoys. Used for static clusters + # defined in Envoy's configuration file + connectTimeout: 10s + # + ### ADVANCED SETTINGS ############# + # Where should envoy's configuration be stored in the istio-proxy container + configPath: "/etc/istio/proxy" + binaryPath: "/usr/local/bin/envoy" + # The pseudo service name used for Envoy. + serviceCluster: istio-proxy + # These settings that determine how long an old Envoy + # process should be kept alive after an occasional reload. + drainDuration: 45s + parentShutdownDuration: 1m0s + # + # The mode used to redirect inbound connections to Envoy. This setting + # has no effect on outbound traffic: iptables REDIRECT is always used for + # outbound connections. + # If "REDIRECT", use iptables REDIRECT to NAT and redirect to Envoy. + # The "REDIRECT" mode loses source addresses during redirection. + # If "TPROXY", use iptables TPROXY to redirect to Envoy. + # The "TPROXY" mode preserves both the source and destination IP + # addresses and ports, so that they can be used for advanced filtering + # and manipulation. + # The "TPROXY" mode also configures the sidecar to run with the + # CAP_NET_ADMIN capability, which is required to use TPROXY. + #interceptionMode: REDIRECT + # + # Port where Envoy listens (on local host) for admin commands + # You can exec into the istio-proxy container in a pod and + # curl the admin port (curl http://localhost:15000/) to obtain + # diagnostic information from Envoy. See + # https://lyft.github.io/envoy/docs/operations/admin.html + # for more details + proxyAdminPort: 15000 + # + # Set concurrency to a specific number to control the number of Proxy worker threads. + # If set to 0 (default), then start worker thread for each CPU thread/core. + concurrency: {{ .Values.global.proxy.concurrency }} + # + {{- if eq .Values.global.proxy.tracer "lightstep" }} + tracing: + lightstep: + # Address of the LightStep Satellite pool + address: {{ .Values.global.tracer.lightstep.address }} + # Access Token used to communicate with the Satellite pool + accessToken: {{ .Values.global.tracer.lightstep.accessToken }} + # Whether communication with the Satellite pool should be secure + secure: {{ .Values.global.tracer.lightstep.secure }} + # Path to the file containing the cacert to use when verifying TLS + cacertPath: {{ .Values.global.tracer.lightstep.cacertPath }} + {{- else if eq .Values.global.proxy.tracer "zipkin" }} + tracing: + zipkin: + # Address of the Zipkin collector + {{- if .Values.global.tracer.zipkin.address }} + address: {{ .Values.global.tracer.zipkin.address }} + {{- else if .Values.global.remoteZipkinAddress }} + address: {{ .Values.global.remoteZipkinAddress }}:9411 + {{- else }} + address: zipkin.{{ .Release.Namespace }}:9411 + {{- end }} + {{- else if eq .Values.global.proxy.tracer "datadog" }} + tracing: + datadog: + # Address of the Datadog Agent + address: {{ .Values.global.tracer.datadog.address }} + {{- else if eq .Values.global.proxy.tracer "stackdriver" }} + tracing: + stackdriver: {} + {{- end }} + + {{- if .Values.global.proxy.envoyStatsd.enabled }} + # + # Statsd metrics collector converts statsd metrics into Prometheus metrics. + statsdUdpAddress: {{ .Values.global.proxy.envoyStatsd.host }}:{{ .Values.global.proxy.envoyStatsd.port }} + {{- end }} + + {{- if .Values.global.proxy.envoyMetricsService.enabled }} + # + # Envoy's Metrics Service stats sink pushes Envoy metrics to a remote collector via the Metrics Service gRPC API. + envoyMetricsService: + address: {{ .Values.global.proxy.envoyMetricsService.host }}:{{ .Values.global.proxy.envoyMetricsService.port }} + {{- end}} + + {{- if .Values.global.proxy.envoyAccessLogService.enabled }} + # + # Envoy's AccessLog Service pushes access logs to a remote collector via the Access Log Service gRPC API. + envoyAccessLogService: + address: {{ .Values.global.proxy.envoyAccessLogService.host }}:{{ .Values.global.proxy.envoyAccessLogService.port }} + {{- if .Values.global.proxy.envoyAccessLogService.tlsSettings }} + tlsSettings: +{{ toYaml .Values.global.proxy.envoyAccessLogService.tlsSettings | indent 10 }} + {{- end}} + {{- if .Values.global.proxy.envoyAccessLogService.tcpKeepalive }} + tcpKeepalive: +{{ toYaml .Values.global.proxy.envoyAccessLogService.tcpKeepalive | indent 10 }} + {{- end}} + {{- end}} + + {{- $defPilotHostname := printf "istio-pilot.%s" .Release.Namespace }} + {{- $pilotAddress := .Values.global.remotePilotAddress | default $defPilotHostname }} + {{- if .Values.global.controlPlaneSecurityEnabled }} + # + # Mutual TLS authentication between sidecars and istio control plane. + controlPlaneAuthPolicy: MUTUAL_TLS + # + # Address where istio Pilot service is running + {{- if or .Values.global.remotePilotCreateSvcEndpoint .Values.global.createRemoteSvcEndpoints }} + discoveryAddress: {{ $defPilotHostname }}:15011 + {{- else }} + discoveryAddress: {{ $pilotAddress }}:15011 + {{- end }} + {{- else }} + # + # Mutual TLS authentication between sidecars and istio control plane. + controlPlaneAuthPolicy: NONE + # + # Address where istio Pilot service is running + {{- if or .Values.global.remotePilotCreateSvcEndpoint .Values.global.createRemoteSvcEndpoints }} + discoveryAddress: {{ $defPilotHostname }}:15010 + {{- else }} + discoveryAddress: {{ $pilotAddress }}:15010 + {{- end }} + {{- end }} + + # Configuration file for the mesh networks to be used by the Split Horizon EDS. + meshNetworks: |- + {{- if .Values.global.meshNetworks }} + networks: +{{ toYaml .Values.global.meshNetworks | indent 6 }} + {{- else }} + networks: {} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/endpoints.yaml b/kubernetes/helm_charts/istio-system/istio/templates/endpoints.yaml new file mode 100755 index 000000000..81b821853 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/endpoints.yaml @@ -0,0 +1,63 @@ +{{- if or .Values.global.remotePilotCreateSvcEndpoint .Values.global.createRemoteSvcEndpoints }} +apiVersion: v1 +kind: Endpoints +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} +subsets: +- addresses: + - ip: {{ .Values.global.remotePilotAddress }} + ports: + - port: 15003 + name: http-old-discovery # mTLS or non-mTLS depending on auth setting + - port: 15005 + name: https-discovery # always mTLS + - port: 15007 + name: http-discovery # always plain-text + - port: 15010 + name: grpc-xds # direct + - port: 15011 + name: https-xds # mTLS or non-mTLS depending on auth setting + - port: 8080 + name: http-legacy-discovery # direct + - port: 15014 + name: http-monitoring +{{- end }} +{{- if and .Values.global.remotePolicyAddress .Values.global.createRemoteSvcEndpoints }} +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: istio-policy + namespace: {{ .Release.Namespace }} +subsets: +- addresses: + - ip: {{ .Values.global.remotePolicyAddress }} + ports: + - name: grpc-mixer + port: 9091 + - name: grpc-mixer-mtls + port: 15004 + - name: http-monitoring + port: 15014 +{{- end }} +{{- if and .Values.global.remoteTelemetryAddress .Values.global.createRemoteSvcEndpoints }} +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: istio-telemetry + namespace: {{ .Release.Namespace }} +subsets: +- addresses: + - ip: {{ .Values.global.remoteTelemetryAddress }} + ports: + - name: grpc-mixer + port: 9091 + - name: grpc-mixer-mtls + port: 15004 + - name: http-monitoring + port: 15014 + - name: prometheus + port: 42422 +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/install-custom-resources.sh.tpl b/kubernetes/helm_charts/istio-system/istio/templates/install-custom-resources.sh.tpl new file mode 100755 index 000000000..a5525a139 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/install-custom-resources.sh.tpl @@ -0,0 +1,32 @@ +{{ define "install-custom-resources.sh.tpl" }} +#!/bin/sh + +set -x + +if [ "$#" -ne "1" ]; then + echo "first argument should be path to custom resource yaml" + exit 1 +fi + +pathToResourceYAML=${1} + +kubectl get validatingwebhookconfiguration istio-galley 2>/dev/null +if [ "$?" -eq 0 ]; then + echo "istio-galley validatingwebhookconfiguration found - waiting for istio-galley deployment to be ready" + while true; do + kubectl -n {{ .Release.Namespace }} get deployment istio-galley 2>/dev/null + if [ "$?" -eq 0 ]; then + break + fi + sleep 1 + done + kubectl -n {{ .Release.Namespace }} rollout status deployment istio-galley + if [ "$?" -ne 0 ]; then + echo "istio-galley deployment rollout status check failed" + exit 1 + fi + echo "istio-galley deployment ready for configuration validation" +fi +sleep 5 +kubectl apply -f ${pathToResourceYAML} +{{ end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/service.yaml b/kubernetes/helm_charts/istio-system/istio/templates/service.yaml new file mode 100755 index 000000000..732cdefd2 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/service.yaml @@ -0,0 +1,60 @@ +{{- if or .Values.global.remotePilotCreateSvcEndpoint .Values.global.createRemoteSvcEndpoints }} +apiVersion: v1 +kind: Service +metadata: + name: istio-pilot + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 15003 + name: http-old-discovery # mTLS or non-mTLS depending on auth setting + - port: 15005 + name: https-discovery # always mTLS + - port: 15007 + name: http-discovery # always plain-text + - port: 15010 + name: grpc-xds # direct + - port: 15011 + name: https-xds # mTLS or non-mTLS depending on auth setting + - port: 8080 + name: http-legacy-discovery # direct + - port: 15014 + name: http-monitoring + clusterIP: None +{{- end }} +{{- if and .Values.global.remotePolicyAddress .Values.global.createRemoteSvcEndpoints }} +--- +apiVersion: v1 +kind: Service +metadata: + name: istio-policy + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: grpc-mixer + port: 9091 + - name: grpc-mixer-mtls + port: 15004 + - name: http-monitoring + port: 15014 + clusterIP: None +{{- end }} +{{- if and .Values.global.remoteTelemetryAddress .Values.global.createRemoteSvcEndpoints }} +--- +apiVersion: v1 +kind: Service +metadata: + name: istio-telemetry + namespace: {{ .Release.Namespace }} +spec: + ports: + - name: grpc-mixer + port: 9091 + - name: grpc-mixer-mtls + port: 15004 + - name: http-monitoring + port: 15014 + - name: prometheus + port: 42422 + clusterIP: None +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/serviceaccount.yaml b/kubernetes/helm_charts/istio-system/istio/templates/serviceaccount.yaml new file mode 100755 index 000000000..e52d9eb9c --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istio-multi + namespace: {{ .Release.Namespace }} diff --git a/kubernetes/helm_charts/istio-system/istio/templates/sidecar-injector-configmap.yaml b/kubernetes/helm_charts/istio-system/istio/templates/sidecar-injector-configmap.yaml new file mode 100755 index 000000000..135b7fbad --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/templates/sidecar-injector-configmap.yaml @@ -0,0 +1,25 @@ +{{- if not .Values.global.omitSidecarInjectorConfigMap }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: istio-sidecar-injector + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "istio.name" . }} + chart: {{ template "istio.chart" . }} + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + istio: sidecar-injector +data: + values: |- + {{ .Values | toJson }} + + config: |- + policy: {{ .Values.global.proxy.autoInject }} + alwaysInjectSelector: +{{ toYaml .Values.sidecarInjectorWebhook.alwaysInjectSelector | trim | indent 6 }} + neverInjectSelector: +{{ toYaml .Values.sidecarInjectorWebhook.neverInjectSelector | trim | indent 6 }} + template: |- +{{ .Files.Get "files/injection-template.yaml" | trim | indent 6 }} +{{- end }} diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-demo-auth.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-demo-auth.yaml new file mode 100755 index 000000000..76abed3a0 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-demo-auth.yaml @@ -0,0 +1,87 @@ +# This is used to generate minimal demo mode. It is included from demo and demo-auth values. +# It is shipped with the release, used for bookinfo or quick installation of istio. +# Includes components used in the demo, defaults to alpha3 rules. +# Note: please only put common configuration for the demo profiles here. +global: + proxy: + accessLogFile: "/dev/stdout" + resources: + requests: + cpu: 10m + memory: 40Mi + + disablePolicyChecks: false + +sidecarInjectorWebhook: + enabled: true + # If true, webhook or istioctl injector will rewrite PodSpec for liveness + # health check to redirect request to sidecar. This makes liveness check work + # even when mTLS is enabled. + rewriteAppHTTPProbe: false + +pilot: + autoscaleEnabled: false + traceSampling: 100.0 + resources: + requests: + cpu: 10m + memory: 100Mi + +mixer: + policy: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 100Mi + + telemetry: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 50m + memory: 100Mi + + adapters: + stdio: + enabled: true + +grafana: + enabled: true + +tracing: + enabled: true + +kiali: + enabled: true + createDemoSecret: true + +gateways: + istio-ingressgateway: + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 40Mi + + istio-egressgateway: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 40Mi +# This is used to generate istio-auth.yaml for minimal, demo mode with MTLS enabled. +# It is shipped with the release, used for bookinfo or quick installation of istio. +# Includes components used in the demo, defaults to alpha3 rules. + +# @include +global: + controlPlaneSecurityEnabled: true + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: true diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-demo.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-demo.yaml new file mode 100755 index 000000000..2a9bf89c6 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-demo.yaml @@ -0,0 +1,88 @@ +# This is used to generate minimal demo mode. It is included from demo and demo-auth values. +# It is shipped with the release, used for bookinfo or quick installation of istio. +# Includes components used in the demo, defaults to alpha3 rules. +# Note: please only put common configuration for the demo profiles here. +global: + proxy: + accessLogFile: "/dev/stdout" + resources: + requests: + cpu: 10m + memory: 40Mi + + disablePolicyChecks: false + +sidecarInjectorWebhook: + enabled: true + # If true, webhook or istioctl injector will rewrite PodSpec for liveness + # health check to redirect request to sidecar. This makes liveness check work + # even when mTLS is enabled. + rewriteAppHTTPProbe: false + +pilot: + autoscaleEnabled: false + traceSampling: 100.0 + resources: + requests: + cpu: 10m + memory: 100Mi + +mixer: + policy: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 100Mi + + telemetry: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 50m + memory: 100Mi + + adapters: + stdio: + enabled: true + +grafana: + enabled: true + +tracing: + enabled: true + +kiali: + enabled: true + createDemoSecret: true + +gateways: + istio-ingressgateway: + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 40Mi + + istio-egressgateway: + enabled: true + autoscaleEnabled: false + resources: + requests: + cpu: 10m + memory: 40Mi +# This is used to generate istio.yaml for minimal, demo mode. +# It is shipped with the release, used for bookinfo or quick installation of istio. +# Includes components used in the demo, defaults to alpha3 rules. + +# @include +# +global: + controlPlaneSecurityEnabled: false + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: false diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-minimal.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-minimal.yaml new file mode 100755 index 000000000..de45fa458 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-minimal.yaml @@ -0,0 +1,46 @@ +# +# Minimal Istio Configuration: https://istio.io/docs/setup/kubernetes/additional-setup/config-profiles/ +# +pilot: + enabled: true + sidecar: false + +gateways: + enabled: false + +security: + enabled: false + +sidecarInjectorWebhook: + enabled: false + +galley: + enabled: false + +mixer: + policy: + enabled: false + telemetry: + enabled: false + +prometheus: + enabled: false + + +# Common settings. +global: + + proxy: + # Sets the destination Statsd in envoy (the value of the "--statsdUdpAddress" proxy argument + # would be :). + # Disabled by default. + # The istio-statsd-prom-bridge is deprecated and should not be used moving forward. + envoyStatsd: + # If enabled is set to true, host and port must also be provided. Istio no longer provides a statsd collector. + enabled: false + host: # example: statsd-svc.istio-system + port: # example: 9125 + + useMCP: false + + diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-remote.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-remote.yaml new file mode 100755 index 000000000..20fe2ac3d --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-remote.yaml @@ -0,0 +1,34 @@ +gateways: + enabled: false + +galley: + enabled: false + +mixer: + policy: + enabled: false + telemetry: + enabled: false + +pilot: + enabled: false + +security: + enabled: true + createMeshPolicy: false + +prometheus: + enabled: false + +global: + istioRemote: true + + enableTracing: false + + # Sets an identifier for the remote network to be used for Split Horizon EDS. The network will be sent + # to the Pilot when connected by the sidecar and will affect the results returned in EDS requests. + # Based on the network identifier Pilot will return all local endpoints + endpoints of gateways to + # other networks. + # + # Must match the names in the meshNetworks section in the Istio local. + network: "" diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth-control-plane-auth-disabled.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth-control-plane-auth-disabled.yaml new file mode 100755 index 000000000..6144aff24 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth-control-plane-auth-disabled.yaml @@ -0,0 +1,21 @@ +global: + controlPlaneSecurityEnabled: false + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: true + + sds: + enabled: true + udsPath: "unix:/var/run/sds/uds_path" + token: + aud: "istio-ca" + +nodeagent: + enabled: true + image: node-agent-k8s + env: + CA_PROVIDER: "Citadel" + CA_ADDR: "istio-citadel:8060" + VALID_TOKEN: true diff --git a/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth.yaml b/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth.yaml new file mode 100755 index 000000000..d6b8c65d4 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values-istio-sds-auth.yaml @@ -0,0 +1,21 @@ +global: + controlPlaneSecurityEnabled: true + + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: true + + sds: + enabled: true + udsPath: "unix:/var/run/sds/uds_path" + token: + aud: "istio-ca" + +nodeagent: + enabled: true + image: node-agent-k8s + env: + CA_PROVIDER: "Citadel" + CA_ADDR: "istio-citadel:8060" + VALID_TOKEN: true \ No newline at end of file diff --git a/kubernetes/helm_charts/istio-system/istio/values.yaml b/kubernetes/helm_charts/istio-system/istio/values.yaml new file mode 100755 index 000000000..072fb7920 --- /dev/null +++ b/kubernetes/helm_charts/istio-system/istio/values.yaml @@ -0,0 +1,563 @@ +# Top level istio values file has the following sections. +# +# global: This file is the authoritative and exhaustive source for the global section. +# +# chart sections: Every subdirectory inside the charts/ directory has a top level +# configuration key in this file. This file overrides the values specified +# by the charts/${chartname}/values.yaml. +# Check the chart level values file for exhaustive list of configuration options. + +# +# Gateways Configuration, refer to the charts/gateways/values.yaml +# for detailed configuration +# +gateways: + enabled: true + +# +# sidecar-injector webhook configuration, refer to the +# charts/sidecarInjectorWebhook/values.yaml for detailed configuration +# +sidecarInjectorWebhook: + enabled: true + +# +# galley configuration, refer to charts/galley/values.yaml +# for detailed configuration +# +galley: + enabled: true + +# +# mixer configuration +# +# @see charts/mixer/values.yaml, it takes precedence +mixer: + policy: + # if policy is enabled the global.disablePolicyChecks has affect. + enabled: true + + telemetry: + enabled: true +# +# pilot configuration +# +# @see charts/pilot/values.yaml +pilot: + enabled: true + +# +# security configuration +# +security: + enabled: true + +# +# nodeagent configuration +# +nodeagent: + enabled: false + +# +# addon grafana configuration +# +grafana: + enabled: false + +# +# addon prometheus configuration +# +prometheus: + enabled: true + +# +# addon jaeger tracing configuration +# +tracing: + enabled: false + +# +# addon kiali tracing configuration +# +kiali: + enabled: false + +# +# addon certmanager configuration +# +certmanager: + enabled: false + +# +# Istio CNI plugin enabled +# This must be enabled to use the CNI plugin in Istio. The CNI plugin is installed separately. +# If true, the privileged initContainer istio-init is not needed to perform the traffic redirect +# settings for the istio-proxy. +# +istio_cni: + enabled: false + +# addon Istio CoreDNS configuration +# +istiocoredns: + enabled: false + +# Common settings used among istio subcharts. +global: + # Default hub for Istio images. + # Releases are published to docker hub under 'istio' project. + # Daily builds from prow are on gcr.io + hub: docker.io/istio + + # Default tag for Istio images. + tag: 1.3.3 + + # Comma-separated minimum per-scope logging level of messages to output, in the form of :,: + # The control plane has different scopes depending on component, but can configure default log level across all components + # If empty, default scope and level will be used as configured in code + logging: + level: "default:info" + + # monitoring port used by mixer, pilot, galley and sidecar injector + monitoringPort: 15014 + + k8sIngress: + enabled: false + # Gateway used for k8s Ingress resources. By default it is + # using 'istio:ingressgateway' that will be installed by setting + # 'gateways.enabled' and 'gateways.istio-ingressgateway.enabled' + # flags to true. + gatewayName: ingressgateway + # enableHttps will add port 443 on the ingress. + # It REQUIRES that the certificates are installed in the + # expected secrets - enabling this option without certificates + # will result in LDS rejection and the ingress will not work. + enableHttps: false + + proxy: + # Configuration for the proxy init container + init: + resources: + limits: + cpu: 100m + memory: 50Mi + requests: + cpu: 10m + memory: 10Mi + image: proxyv2 + + # cluster domain. Default value is "cluster.local". + clusterDomain: "cluster.local" + + # Resources for the sidecar. + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 2000m + memory: 1024Mi + + # Controls number of Proxy worker threads. + # If set to 0, then start worker thread for each CPU thread/core. + concurrency: 2 + + # Configures the access log for each sidecar. + # Options: + # "" - disables access log + # "/dev/stdout" - enables access log + accessLogFile: "" + + # Configure how and what fields are displayed in sidecar access log. Setting to + # empty string will result in default log format + accessLogFormat: "[%START_TIME%] \"%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%\" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% \"%REQ(USER-AGENT)%\" \"%REQ(X-REQUEST-ID)%\" \"%REQ(:AUTHORITY)%\" \"%UPSTREAM_HOST%\" \"%REQ(X-DEVICE-ID)%\" \"%REQ(X-CHANNEL-ID)%\" \"%REQ(X-APP-ID)%\"\n" + # Configure the access log for sidecar to JSON or TEXT. + accessLogEncoding: TEXT + + # Configure envoy gRPC access log service. + envoyAccessLogService: + enabled: false + host: # example: accesslog-service.istio-system + port: # example: 15000 + tlsSettings: + mode: DISABLE # DISABLE, SIMPLE, MUTUAL, ISTIO_MUTUAL + clientCertificate: # example: /etc/istio/als/cert-chain.pem + privateKey: # example: /etc/istio/als/key.pem + caCertificates: # example: /etc/istio/als/root-cert.pem + sni: # example: als.somedomain + subjectAltNames: [] + # - als.somedomain + tcpKeepalive: + probes: 3 + time: 10s + interval: 10s + + # Log level for proxy, applies to gateways and sidecars. If left empty, "warning" is used. + # Expected values are: trace|debug|info|warning|error|critical|off + logLevel: "" + + # Per Component log level for proxy, applies to gateways and sidecars. If a component level is + # not set, then the global "logLevel" will be used. If left empty, "misc:error" is used. + componentLogLevel: "" + + # Configure the DNS refresh rate for Envoy cluster of type STRICT_DNS + # This must be given it terms of seconds. For example, 300s is valid but 5m is invalid. + dnsRefreshRate: 300s + + # Automatic protocol detection uses a set of heuristics to + # determine whether the connection is using TLS or not (on the + # server side), as well as the application protocol being used + # (e.g., http vs tcp). These heuristics rely on the client sending + # the first bits of data. For server first protocols like MySQL, + # MongoDB, etc., Envoy will timeout on the protocol detection after + # the specified period, defaulting to non mTLS plain TCP + # traffic. Set this field to tweak the period that Envoy will wait + # for the client to send the first bits of data. (MUST BE >=1ms) + protocolDetectionTimeout: 100ms + + #If set to true, istio-proxy container will have privileged securityContext + privileged: false + + # If set, newly injected sidecars will have core dumps enabled. + enableCoreDump: false + + # Image used to enable core dumps. This is only used, when "enableCoreDump" is set to true. + enableCoreDumpImage: ubuntu:xenial + + # Default port for Pilot agent health checks. A value of 0 will disable health checking. + statusPort: 15020 + + # The initial delay for readiness probes in seconds. + readinessInitialDelaySeconds: 1 + + # The period between readiness probes. + readinessPeriodSeconds: 2 + + # The number of successive failed probes before indicating readiness failure. + readinessFailureThreshold: 30 + + # istio egress capture whitelist + # https://istio.io/docs/tasks/traffic-management/egress.html#calling-external-services-directly + # example: includeIPRanges: "172.30.0.0/16,172.20.0.0/16" + # would only capture egress traffic on those two IP Ranges, all other outbound traffic would + # be allowed by the sidecar + includeIPRanges: "*" + excludeIPRanges: "" + excludeOutboundPorts: "" + + # pod internal interfaces + kubevirtInterfaces: "" + + # istio ingress capture whitelist + # examples: + # Redirect no inbound traffic to Envoy: --includeInboundPorts="" + # Redirect all inbound traffic to Envoy: --includeInboundPorts="*" + # Redirect only selected ports: --includeInboundPorts="80,8080" + includeInboundPorts: "*" + excludeInboundPorts: "" + + # This controls the 'policy' in the sidecar injector. + autoInject: enabled + + # Sets the destination Statsd in envoy (the value of the "--statsdUdpAddress" proxy argument + # would be :). + # Disabled by default. + # The istio-statsd-prom-bridge is deprecated and should not be used moving forward. + envoyStatsd: + # If enabled is set to true, host and port must also be provided. Istio no longer provides a statsd collector. + enabled: false + host: # example: statsd-svc.istio-system + port: # example: 9125 + + # Sets the Envoy Metrics Service address, used to push Envoy metrics to an external collector + # via the Metrics Service gRPC API. This contains detailed stats information emitted directly + # by Envoy and should not be confused with the the Istio telemetry. The Envoy stats are also + # available to scrape via the Envoy admin port at either /stats or /stats/prometheus. + # + # See https://www.envoyproxy.io/docs/envoy/latest/api-v2/config/metrics/v2/metrics_service.proto + # for details about Envoy's Metrics Service API. + # + # Disabled by default. + envoyMetricsService: + enabled: false + host: # example: metrics-service.istio-system + port: # example: 15000 + + # Specify which tracer to use. One of: zipkin, lightstep, datadog, stackdriver. + # If using stackdriver tracer outside GCP, set env GOOGLE_APPLICATION_CREDENTIALS to the GCP credential file. + tracer: "zipkin" + + proxy_init: + # Base name for the proxy_init container, used to configure iptables. + image: proxy_init + + # imagePullPolicy is applied to istio control plane components. + # local tests require IfNotPresent, to avoid uploading to dockerhub. + # TODO: Switch to Always as default, and override in the local tests. + imagePullPolicy: IfNotPresent + + # controlPlaneSecurityEnabled enabled. Will result in delays starting the pods while secrets are + # propagated, not recommended for tests. + controlPlaneSecurityEnabled: false + + # disablePolicyChecks disables mixer policy checks. + # if mixer.policy.enabled==true then disablePolicyChecks has affect. + # Will set the value with same name in istio config map - pilot needs to be restarted to take effect. + disablePolicyChecks: true + + # policyCheckFailOpen allows traffic in cases when the mixer policy service cannot be reached. + # Default is false which means the traffic is denied when the client is unable to connect to Mixer. + policyCheckFailOpen: false + + # EnableTracing sets the value with same name in istio config map, requires pilot restart to take effect. + enableTracing: true + + # Configuration for each of the supported tracers + tracer: + # Configuration for envoy to send trace data to LightStep. + # Disabled by default. + # address: the : of the satellite pool + # accessToken: required for sending data to the pool + # secure: specifies whether data should be sent with TLS + # cacertPath: the path to the file containing the cacert to use when verifying TLS. If secure is true, this is + # required. If a value is specified then a secret called "lightstep.cacert" must be created in the destination + # namespace with the key matching the base of the provided cacertPath and the value being the cacert itself. + # + lightstep: + address: "" # example: lightstep-satellite:443 + accessToken: "" # example: abcdefg1234567 + secure: true # example: true|false + cacertPath: "" # example: /etc/lightstep/cacert.pem + zipkin: + # Host:Port for reporting trace data in zipkin format. If not specified, will default to + # zipkin service (port 9411) in the same namespace as the other istio components. + address: "" + datadog: + # Host:Port for submitting traces to the Datadog agent. + address: "$(HOST_IP):8126" + + # Default mtls policy. If true, mtls between services will be enabled by default. + mtls: + # Default setting for service-to-service mtls. Can be set explicitly using + # destination rules or service annotations. + enabled: false + + # Lists the secrets you need to use to pull Istio images from a private registry. + imagePullSecrets: [] + # - private-registry-key + + # Specify pod scheduling arch(amd64, ppc64le, s390x) and weight as follows: + # 0 - Never scheduled + # 1 - Least preferred + # 2 - No preference + # 3 - Most preferred + arch: + amd64: 2 + s390x: 2 + ppc64le: 2 + + # Whether to restrict the applications namespace the controller manages; + # If not set, controller watches all namespaces + oneNamespace: false + + # Default node selector to be applied to all deployments so that all pods can be + # constrained to run a particular nodes. Each component can overwrite these default + # values by adding its node selector block in the relevant section below and setting + # the desired values. + defaultNodeSelector: {} + + # Default node tolerations to be applied to all deployments so that all pods can be + # scheduled to a particular nodes with matching taints. Each component can overwrite + # these default values by adding its tolerations block in the relevant section below + # and setting the desired values. + # Configure this field in case that all pods of Istio control plane are expected to + # be scheduled to particular nodes with specified taints. + defaultTolerations: [] + + # Whether to perform server-side validation of configuration. + configValidation: true + + # Custom DNS config for the pod to resolve names of services in other + # clusters. Use this to add additional search domains, and other settings. + # see + # https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#dns-config + # This does not apply to gateway pods as they typically need a different + # set of DNS settings than the normal application pods (e.g., in + # multicluster scenarios). + # NOTE: If using templates, follow the pattern in the commented example below. + # podDNSSearchNamespaces: + # - global + # - "[[ valueOrDefault .DeploymentMeta.Namespace \"default\" ]].global" + + # If set to true, the pilot and citadel mtls will be exposed on the + # ingress gateway + meshExpansion: + enabled: false + # If set to true, the pilot and citadel mtls and the plaintext pilot ports + # will be exposed on an internal gateway + useILB: false + + multiCluster: + # Set to true to connect two kubernetes clusters via their respective + # ingressgateway services when pods in each cluster cannot directly + # talk to one another. All clusters should be using Istio mTLS and must + # have a shared root CA for this model to work. + enabled: false + + # Should be set to the name of the cluster this installation will run in. This is required for sidecar injection + # to properly label proxies + clusterName: "" + + # A minimal set of requested resources to applied to all deployments so that + # Horizontal Pod Autoscaler will be able to function (if set). + # Each component can overwrite these default values by adding its own resources + # block in the relevant section below and setting the desired resources values. + defaultResources: + requests: + cpu: 10m + # memory: 128Mi + # limits: + # cpu: 100m + # memory: 128Mi + + # enable pod distruption budget for the control plane, which is used to + # ensure Istio control plane components are gradually upgraded or recovered. + defaultPodDisruptionBudget: + enabled: true + # The values aren't mutable due to a current PodDisruptionBudget limitation + # minAvailable: 1 + + # Kubernetes >=v1.11.0 will create two PriorityClass, including system-cluster-critical and + # system-node-critical, it is better to configure this in order to make sure your Istio pods + # will not be killed because of low priority class. + # Refer to https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass + # for more detail. + priorityClassName: "" + + # Use the Mesh Control Protocol (MCP) for configuring Mixer and + # Pilot. Requires galley (`--set galley.enabled=true`). + useMCP: true + + # The trust domain corresponds to the trust root of a system + # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain + # Indicate the domain used in SPIFFE identity URL + # The default depends on the environment. + # kubernetes: cluster.local + # else: default dns domain + trustDomain: "" + + # Mesh ID means Mesh Identifier. It should be unique within the scope where + # meshes will interact with each other, but it is not required to be + # globally/universally unique. For example, if any of the following are true, + # then two meshes must have different Mesh IDs: + # - Meshes will have their telemetry aggregated in one place + # - Meshes will be federated together + # - Policy will be written referencing one mesh from the other + # + # If an administrator expects that any of these conditions may become true in + # the future, they should ensure their meshes have different Mesh IDs + # assigned. + # + # Within a multicluster mesh, each cluster must be (manually or auto) + # configured to have the same Mesh ID value. If an existing cluster 'joins' a + # multicluster mesh, it will need to be migrated to the new mesh ID. Details + # of migration TBD, and it may be a disruptive operation to change the Mesh + # ID post-install. + # + # If the mesh admin does not specify a value, Istio will use the value of the + # mesh's Trust Domain. The best practice is to select a proper Trust Domain + # value. + meshID: "" + + # Set the default behavior of the sidecar for handling outbound traffic from the application: + # ALLOW_ANY - outbound traffic to unknown destinations will be allowed, in case there are no + # services or ServiceEntries for the destination port + # REGISTRY_ONLY - restrict outbound traffic to services defined in the service registry as well + # as those defined through ServiceEntries + # ALLOW_ANY is the default in 1.1. This means each pod will be able to make outbound requests + # to services outside of the mesh without any ServiceEntry. + # REGISTRY_ONLY was the default in 1.0. If this behavior is desired, set the value below to REGISTRY_ONLY. + outboundTrafficPolicy: + mode: ALLOW_ANY + + # The namespace where globally shared configurations should be present. + # DestinationRules that apply to the entire mesh (e.g., enabling mTLS), + # default Sidecar configs, etc. should be added to this namespace. + # configRootNamespace: istio-config + + # set the default set of namespaces to which services, service entries, virtual services, destination + # rules should be exported to. Currently only one value can be provided in this list. This value + # should be one of the following two options: + # * implies these objects are visible to all namespaces, enabling any sidecar to talk to any other sidecar. + # . implies these objects are visible to only to sidecars in the same namespace, or if imported as a Sidecar.egress.host + # defaultConfigVisibilitySettings: + #- '*' + + sds: + # SDS enabled. IF set to true, mTLS certificates for the sidecars will be + # distributed through the SecretDiscoveryService instead of using K8S secrets to mount the certificates. + enabled: false + udsPath: "" + # The JWT token for SDS and the aud field of such JWT. See RFC 7519, section 4.1.3. + # When a CSR is sent from Citadel Agent to the CA (e.g. Citadel), this aud is to make sure the + # JWT is intended for the CA. + token: + aud: istio-ca + + # Configure the mesh networks to be used by the Split Horizon EDS. + # + # The following example defines two networks with different endpoints association methods. + # For `network1` all endpoints that their IP belongs to the provided CIDR range will be + # mapped to network1. The gateway for this network example is specified by its public IP + # address and port. + # The second network, `network2`, in this example is defined differently with all endpoints + # retrieved through the specified Multi-Cluster registry being mapped to network2. The + # gateway is also defined differently with the name of the gateway service on the remote + # cluster. The public IP for the gateway will be determined from that remote service (only + # LoadBalancer gateway service type is currently supported, for a NodePort type gateway service, + # it still need to be configured manually). + # + # meshNetworks: + # network1: + # endpoints: + # - fromCidr: "192.168.0.1/24" + # gateways: + # - address: 1.1.1.1 + # port: 80 + # network2: + # endpoints: + # - fromRegistry: reg1 + # gateways: + # - registryServiceName: istio-ingressgateway.istio-system.svc.cluster.local + # port: 443 + # + meshNetworks: {} + + # Specifies the global locality load balancing settings. + # Locality-weighted load balancing allows administrators to control the distribution of traffic to + # endpoints based on the localities of where the traffic originates and where it will terminate. + # Either failover or distribute configuration can be set, but not both. If neither are provided + # failover mode will be used. + # + # localityLbSetting: + # enabled: true + # distribute: + # - from: "us-central1/*" + # to: + # "us-central1/*": 80 + # "us-central2/*": 20 + # + # localityLbSetting: + # enabled: true + # failover: + # - from: us-east + # to: eu-west + # - from: us-west + # to: us-east + localityLbSetting: + enabled: true + + # Specifies whether helm test is enabled or not. + # This field is set to false by default, so 'helm template ...' + # will ignore the helm test yaml files when generating the template + enableHelmTest: false diff --git a/kubernetes/helm_charts/kube-system/coredns-custom.yaml b/kubernetes/helm_charts/kube-system/coredns-custom.yaml new file mode 100644 index 000000000..d3a0f068e --- /dev/null +++ b/kubernetes/helm_charts/kube-system/coredns-custom.yaml @@ -0,0 +1,23 @@ +# enabling istioctl serviceentries to create dns entries +# +# resolves *.sunbird to istiocoredns +# +# TODO: is there any way to change the ip to domain +# of istiodns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + addonmanager.kubernetes.io/mode: EnsureExists + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + name: coredns-custom + namespace: kube-system +data: + Sunbird.server: |- + global:53 { + errors + cache 30 + forward . 172.16.242.123 + } diff --git a/kubernetes/helm_charts/logging/elasticsearch/.helmignore b/kubernetes/helm_charts/logging/elasticsearch/.helmignore new file mode 100755 index 000000000..e12c0b4b9 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/.helmignore @@ -0,0 +1,2 @@ +tests/ +.pytest_cache/ diff --git a/kubernetes/helm_charts/logging/elasticsearch/Chart.yaml b/kubernetes/helm_charts/logging/elasticsearch/Chart.yaml new file mode 100755 index 000000000..cfb40d891 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/Chart.yaml @@ -0,0 +1,11 @@ +description: Official Elastic helm chart for Elasticsearch +home: https://github.com/elastic/helm-charts +maintainers: +- email: helm-charts@elastic.co + name: Elastic +name: elasticsearch +version: 7.3.2 +appVersion: 7.3.2 +sources: + - https://github.com/elastic/elasticsearch +icon: https://helm.elastic.co/icons/elasticsearch.png diff --git a/kubernetes/helm_charts/logging/elasticsearch/Makefile b/kubernetes/helm_charts/logging/elasticsearch/Makefile new file mode 100755 index 000000000..22218a1f6 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/Makefile @@ -0,0 +1 @@ +include ../helpers/common.mk diff --git a/kubernetes/helm_charts/logging/elasticsearch/README.md b/kubernetes/helm_charts/logging/elasticsearch/README.md new file mode 100755 index 000000000..ff90f09c0 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/README.md @@ -0,0 +1,307 @@ +# Elasticsearch Helm Chart + +This functionality is in beta and is subject to change. The design and code is less mature than official GA features and is being provided as-is with no warranties. Beta features are not subject to the support SLA of official GA features. + +This helm chart is a lightweight way to configure and run our official [Elasticsearch docker image](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) + +## Requirements + +* [Helm](https://helm.sh/) >= 2.8.0 +* Kubernetes >= 1.8 +* Minimum cluster requirements include the following to run this chart with default settings. All of these settings are configurable. + * Three Kubernetes nodes to respect the default "hard" affinity settings + * 1GB of RAM for the JVM heap + +## Usage notes and getting started + +* This repo includes a number of [example](./examples) configurations which can be used as a reference. They are also used in the automated testing of this chart +* Automated testing of this chart is currently only run against GKE (Google Kubernetes Engine). If you are using a different Kubernetes provider you will likely need to adjust the `storageClassName` in the `volumeClaimTemplate` +* The default storage class for GKE is `standard` which by default will give you `pd-ssd` type persistent volumes. This is network attached storage and will not perform as well as local storage. If you are using Kubernetes version 1.10 or greater you can use [Local PersistentVolumes](https://cloud.google.com/kubernetes-engine/docs/how-to/persistent-volumes/local-ssd) for increased performance +* The chart deploys a statefulset and by default will do an automated rolling update of your cluster. It does this by waiting for the cluster health to become green after each instance is updated. If you prefer to update manually you can set [`updateStrategy: OnDelete`](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#on-delete) +* It is important to verify that the JVM heap size in `esJavaOpts` and to set the CPU/Memory `resources` to something suitable for your cluster +* To simplify chart and maintenance each set of node groups is deployed as a separate helm release. Take a look at the [multi](./examples/multi) example to get an idea for how this works. Without doing this it isn't possible to resize persistent volumes in a statefulset. By setting it up this way it makes it possible to add more nodes with a new storage size then drain the old ones. It also solves the problem of allowing the user to determine which node groups to update first when doing upgrades or changes. +* We have designed this chart to be very un-opinionated about how to configure Elasticsearch. It exposes ways to set environment variables and mount secrets inside of the container. Doing this makes it much easier for this chart to support multiple versions with minimal changes. + +## Migration from helm/charts stable + +If you currently have a cluster deployed with the [helm/charts stable](https://github.com/helm/charts/tree/master/stable/elasticsearch) chart you can follow the [migration guide](/elasticsearch/examples/migration/README.md) + +## Installing + +* Add the elastic helm charts repo + ``` + helm repo add elastic https://helm.elastic.co + ``` +* Install it + ``` + helm install --name elasticsearch elastic/elasticsearch + ``` + +## Compatibility + +This chart is tested with the latest supported versions. The currently tested versions are: + +| 6.x | 7.x | +| ----- | ----- | +| 6.8.3 | 7.3.2 | + +Examples of installing older major versions can be found in the [examples](./examples) directory. + +While only the latest releases are tested, it is possible to easily install old or new releases by overriding the `imageTag`. To install version `7.3.2` of Elasticsearch it would look like this: + +``` +helm install --name elasticsearch elastic/elasticsearch --set imageTag=7.3.2 +``` + +## Configuration + +| Parameter | Description | Default | +| ----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | +| `clusterName` | This will be used as the Elasticsearch [cluster.name](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster.name.html) and should be unique per cluster in the namespace | `elasticsearch` | +| `nodeGroup` | This is the name that will be used for each group of nodes in the cluster. The name will be `clusterName-nodeGroup-X` | `master` | +| `masterService` | Optional. The service name used to connect to the masters. You only need to set this if your master `nodeGroup` is set to something other than `master`. See [Clustering and Node Discovery](#clustering-and-node-discovery) for more information. | `` | +| `roles` | A hash map with the [specific roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for the node group | `master: true`
`data: true`
`ingest: true` | +| `replicas` | Kubernetes replica count for the statefulset (i.e. how many pods) | `3` | +| `minimumMasterNodes` | The value for [discovery.zen.minimum_master_nodes](https://www.elastic.co/guide/en/elasticsearch/reference/6.7/discovery-settings.html#minimum_master_nodes). Should be set to `(master_eligible_nodes / 2) + 1`. Ignored in Elasticsearch versions >= 7. | `2` | +| `esMajorVersion` | Used to set major version specific configuration. If you are using a custom image and not running the default Elasticsearch version you will need to set this to the version you are running (e.g. `esMajorVersion: 6`) | `""` | +| `esConfig` | Allows you to add any config files in `/usr/share/elasticsearch/config/` such as `elasticsearch.yml` and `log4j2.properties`. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `extraEnvs` | Extra [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config) which will be appended to the `env:` definition for the container | `[]` | +| `extraVolumes` | Templatable string of additional volumes to be passed to the `tpl` function | `""` | +| `extraVolumeMounts` | Templatable string of additional volumeMounts to be passed to the `tpl` function | `""` | +| `extraInitContainers` | Templatable string of additional init containers to be passed to the `tpl` function | `""` | +| `secretMounts` | Allows you easily mount a secret as a file inside the statefulset. Useful for mounting certificates and other secrets. See [values.yaml](./values.yaml) for an example | `[]` | +| `image` | The Elasticsearch docker image | `docker.elastic.co/elasticsearch/elasticsearch` | +| `imageTag` | The Elasticsearch docker image tag | `7.3.2` | +| `imagePullPolicy` | The Kubernetes [imagePullPolicy](https://kubernetes.io/docs/concepts/containers/images/#updating-images) value | `IfNotPresent` | +| `podAnnotations` | Configurable [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) applied to all Elasticsearch pods | `{}` | +| `labels` | Configurable [label](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) applied to all Elasticsearch pods | `{}` | +| `esJavaOpts` | [Java options](https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html) for Elasticsearch. This is where you should configure the [jvm heap size](https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html) | `-Xmx1g -Xms1g` | +| `resources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the statefulset | `requests.cpu: 100m`
`requests.memory: 2Gi`
`limits.cpu: 1000m`
`limits.memory: 2Gi` | +| `initResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the initContainer in the statefulset | {} | +| `sidecarResources` | Allows you to set the [resources](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/) for the sidecar containers in the statefulset | {} | +| `networkHost` | Value for the [network.host Elasticsearch setting](https://www.elastic.co/guide/en/elasticsearch/reference/current/network.host.html) | `0.0.0.0` | +| `volumeClaimTemplate` | Configuration for the [volumeClaimTemplate for statefulsets](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#stable-storage). You will want to adjust the storage (default `30Gi`) and the `storageClassName` if you are using a different storage class | `accessModes: [ "ReadWriteOnce" ]`
`resources.requests.storage: 30Gi` | +| `persistence.annotations` | Additional persistence annotations for the `volumeClaimTemplate` | `{}` | +| `persistence.enabled` | Enables a persistent volume for Elasticsearch data. Can be disabled for nodes that only have [roles](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) which don't require persistent data. | `true` | +| `priorityClassName` | The [name of the PriorityClass](https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass). No default is supplied as the PriorityClass must be created first. | `""` | +| `antiAffinityTopologyKey` | The [anti-affinity topology key](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). By default this will prevent multiple Elasticsearch nodes from running on the same Kubernetes node | `kubernetes.io/hostname` | +| `antiAffinity` | Setting this to hard enforces the [anti-affinity rules](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity). If it is set to soft it will be done "best effort". Other values will be ignored. | `hard` | +| `nodeAffinity` | Value for the [node affinity settings](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature) | `{}` | +| `podManagementPolicy` | By default Kubernetes [deploys statefulsets serially](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-management-policies). This deploys them in parallel so that they can discover eachother | `Parallel` | +| `protocol` | The protocol that will be used for the readinessProbe. Change this to `https` if you have `xpack.security.http.ssl.enabled` set | `http` | +| `httpPort` | The http port that Kubernetes will use for the healthchecks and the service. If you change this you will also need to set [http.port](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#_settings) in `extraEnvs` | `9200` | +| `transportPort` | The transport port that Kubernetes will use for the service. If you change this you will also need to set [transport port configuration](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html#_transport_settings) in `extraEnvs` | `9300` | +| `service.type` | Type of elasticsearch service. [Service Types](https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types) | `ClusterIP` | +| `service.nodePort` | Custom [nodePort](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) port that can be set if you are using `service.type: nodePort`. | `` | +| `service.annotations` | Annotations that Kubernetes will use for the service. This will configure load balancer if `service.type` is `LoadBalancer` [Annotations](https://kubernetes.io/docs/concepts/services-networking/service/#ssl-support-on-aws) | `{}` | +| `service.httpPortName` | The name of the http port within the service | `http` | +| `service.transportPortName` | The name of the transport port within the service | `transport` | +| `updateStrategy` | The [updateStrategy](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets) for the statefulset. By default Kubernetes will wait for the cluster to be green after upgrading each pod. Setting this to `OnDelete` will allow you to manually delete each pod during upgrades | `RollingUpdate` | +| `maxUnavailable` | The [maxUnavailable](https://kubernetes.io/docs/tasks/run-application/configure-pdb/#specifying-a-poddisruptionbudget) value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | +| `fsGroup (DEPRECATED)` | The Group ID (GID) for [securityContext.fsGroup](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) so that the Elasticsearch user can read from the persistent volume | `` | +| `podSecurityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod) for the pod | `fsGroup: 1000`
`runAsUser: 1000` | +| `securityContext` | Allows you to set the [securityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container) for the container | `capabilities.drop:[ALL]`
`runAsNonRoot: true`
`runAsUser: 1000` | +| `terminationGracePeriod` | The [terminationGracePeriod](https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods) in seconds used when trying to stop the pod | `120` | +| `sysctlInitContainer.enabled` | Allows you to disable the sysctlInitContainer if you are setting vm.max_map_count with another method | `true` | +| `sysctlVmMaxMapCount` | Sets the [sysctl vm.max_map_count](https://www.elastic.co/guide/en/elasticsearch/reference/current/vm-max-map-count.html#vm-max-map-count) needed for Elasticsearch | `262144` | +| `readinessProbe` | Configuration fields for the [readinessProbe](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/) | `failureThreshold: 3`
`initialDelaySeconds: 10`
`periodSeconds: 10`
`successThreshold: 3`
`timeoutSeconds: 5` | +| `clusterHealthCheckParams` | The [Elasticsearch cluster health status params](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params) that will be used by readinessProbe command | `wait_for_status=green&timeout=1s` | +| `imagePullSecrets` | Configuration for [imagePullSecrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-pod-that-uses-your-secret) so that you can use a private registry for your image | `[]` | +| `nodeSelector` | Configurable [nodeSelector](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) so that you can target specific nodes for your Elasticsearch cluster | `{}` | +| `tolerations` | Configurable [tolerations](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) | `[]` | +| `ingress` | Configurable [ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) to expose the Elasticsearch service. See [`values.yaml`](./values.yaml) for an example | `enabled: false` | +| `schedulerName` | Name of the [alternate scheduler](https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/#specify-schedulers-for-pods) | `nil` | +| `masterTerminationFix` | A workaround needed for Elasticsearch < 7.2 to prevent master status being lost during restarts [#63](https://github.com/elastic/helm-charts/issues/63) | `false` | +| `lifecycle` | Allows you to add lifecycle configuration. See [values.yaml](./values.yaml) for an example of the formatting. | `{}` | +| `keystore` | Allows you map Kubernetes secrets into the keystore. See the [config example](/elasticsearch/examples/config/values.yaml) and [how to use the keystore](#how-to-use-the-keystore) | `[]` | +| `rbac` | Configuration for creating a role, role binding and service account as part of this helm chart with `create: true`. Also can be used to reference an external service account with `serviceAccountName: "externalServiceAccountName"`. | `create: false`
`serviceAccountName: ""` | +| `podSecurityPolicy` | Configuration for create a pod security policy with minimal permissions to run this Helm chart with `create: true`. Also can be used to reference an external pod security policy with `name: "externalPodSecurityPolicy"` | `create: false`
`name: ""` | + +## Try it out + +In [examples/](./examples) you will find some example configurations. These examples are used for the automated testing of this helm chart + +### Default + +To deploy a cluster with all default values and run the integration tests + +``` +cd examples/default +make +``` + +### Multi + +A cluster with dedicated node types + +``` +cd examples/multi +make +``` + +### Security + +A cluster with node to node security and https enabled. This example uses autogenerated certificates and password, for a production deployment you want to generate SSL certificates following the [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html#node-certificates). + +* Generate the certificates and install Elasticsearch + ``` + cd examples/security + make + + # Run a curl command to interact with the cluster + kubectl exec -ti security-master-0 -- sh -c 'curl -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD -k https://localhost:9200/_cluster/health?pretty' + ``` + +### FAQ + +#### How to install plugins? + +The [recommended](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#_c_customized_image) way to install plugins into our docker images is to create a custom docker image. + +The Dockerfile would look something like: + +``` +ARG elasticsearch_version +FROM docker.elastic.co/elasticsearch/elasticsearch:${elasticsearch_version} + +RUN bin/elasticsearch-plugin install --batch repository-gcs +``` + +And then updating the `image` in values to point to your custom image. + +There are a couple reasons we recommend this. + +1. Tying the availability of Elasticsearch to the download service to install plugins is not a great idea or something that we recommend. Especially in Kubernetes where it is normal and expected for a container to be moved to another host at random times. +2. Mutating the state of a running docker image (by installing plugins) goes against best practices of containers and immutable infrastructure. + +#### How to use the keystore? + + +##### Basic example + +Create the secret, the key name needs to be the keystore key path. In this example we will create a secret from a file and from a literal string. + +``` +kubectl create secret generic encryption_key --from-file=xpack.watcher.encryption_key=./watcher_encryption_key +kubectl create secret generic slack_hook --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +To add these secrets to the keystore: +``` +keystore: + - secretName: encryption_key + - secretName: slack_hook +``` + +##### Multiple keys + +All keys in the secret will be added to the keystore. To create the previous example in one secret you could also do: + +``` +kubectl create secret generic keystore_secrets --from-file=xpack.watcher.encryption_key=./watcher_encryption_key --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +``` +keystore: + - secretName: keystore_secrets +``` + +##### Custom paths and keys + +If you are using these secrets for other applications (besides the Elasticsearch keystore) then it is also possible to specify the keystore path and which keys you want to add. Everything specified under each `keystore` item will be passed through to the `volumeMounts` section for [mounting the secret](https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets). In this example we will only add the `slack_hook` key from a secret that also has other keys. Our secret looks like this: + +``` +kubectl create secret generic slack_secrets --from-literal=slack_channel='#general' --from-literal=slack_hook='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' +``` + +We only want to add the `slack_hook` key to the keystore at path `xpack.notification.slack.account.monitoring.secure_url`. + +``` +keystore: + - secretName: slack_secrets + items: + - key: slack_hook + path: xpack.notification.slack.account.monitoring.secure_url +``` + +You can also take a look at the [config example](/elasticsearch/examples/config/) which is used as part of the automated testing pipeline. + +#### How to enable snapshotting? + +1. Install your [snapshot plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository.html) into a custom docker image following the [how to install plugins guide](/elasticsearch/README.md#how-to-install-plugins) +2. Add any required secrets or credentials into an Elasticsearch keystore following the [how to use the keystore guide](/elasticsearch/README.md#how-to-use-the-keystore) +3. Configure the [snapshot repository](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html) as you normally would. +4. To automate snapshots you can use a tool like [curator](https://www.elastic.co/guide/en/elasticsearch/client/curator/current/snapshot.html). In the future there are plans to have Elasticsearch manage automated snapshots with [Snapshot Lifecycle Management](https://github.com/elastic/elasticsearch/issues/38461). + +### Local development environments + +This chart is designed to run on production scale Kubernetes clusters with multiple nodes, lots of memory and persistent storage. For that reason it can be a bit tricky to run them against local Kubernetes environments such as minikube. Below are some examples of how to get this working locally. + +#### Minikube + +This chart also works successfully on [minikube](https://kubernetes.io/docs/setup/minikube/) in addition to typical hosted Kubernetes environments. +An example `values.yaml` file for minikube is provided under `examples/`. + +In order to properly support the required persistent volume claims for the Elasticsearch `StatefulSet`, the `default-storageclass` and `storage-provisioner` minikube addons must be enabled. + +``` +minikube addons enable default-storageclass +minikube addons enable storage-provisioner +cd examples/minikube +make +``` + +Note that if `helm` or `kubectl` timeouts occur, you may consider creating a minikube VM with more CPU cores or memory allocated. + +#### Docker for Mac - Kubernetes + +It is also possible to run this chart with the built in Kubernetes cluster that comes with [docker-for-mac](https://docs.docker.com/docker-for-mac/kubernetes/). + +``` +cd examples/docker-for-mac +make +``` + +## Clustering and Node Discovery + +This chart facilitates Elasticsearch node discovery and services by creating two `Service` definitions in Kubernetes, one with the name `$clusterName-$nodeGroup` and another named `$clusterName-$nodeGroup-headless`. +Only `Ready` pods are a part of the `$clusterName-$nodeGroup` service, while all pods (`Ready` or not) are a part of `$clusterName-$nodeGroup-headless`. + +If your group of master nodes has the default `nodeGroup: master` then you can just add new groups of nodes with a different `nodeGroup` and they will automatically discover the correct master. If your master nodes have a different `nodeGroup` name then you will need to set `masterService` to `$clusterName-$masterNodeGroup`. + +The chart value for `masterService` is used to populate `discovery.zen.ping.unicast.hosts`, which Elasticsearch nodes will use to contact master nodes and form a cluster. +Therefore, to add a group of nodes to an existing cluster, setting `masterService` to the desired `Service` name of the related cluster is sufficient. + +For an example of deploying both a group master nodes and data nodes using multiple releases of this chart, see the accompanying values files in `examples/multi`. + +## Testing + +This chart uses [pytest](https://docs.pytest.org/en/latest/) to test the templating logic. The dependencies for testing can be installed from the [`requirements.txt`](../requirements.txt) in the parent directory. + +``` +pip install -r ../requirements.txt +make pytest +``` + +You can also use `helm template` to look at the YAML being generated + +``` +make template +``` + +It is possible to run all of the tests and linting inside of a docker container + +``` +make test +``` + +## Integration Testing + +Integration tests are run using [goss](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md) which is a serverspec like tool written in golang. See [goss.yaml](examples/default/test/goss.yaml) for an example of what the tests look like. + +To run the goss tests against the default example: + +``` +cd examples/default +make goss +``` diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/Makefile new file mode 100755 index 000000000..2020d4a04 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/Makefile @@ -0,0 +1,15 @@ +default: test +include ../../../helpers/examples.mk + +RELEASE := helm-es-six + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + +restart: + helm upgrade --set terminationGracePeriod=121 --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + +test: install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/test/goss.yaml new file mode 100755 index 000000000..b3e98b1c1 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/test/goss.yaml @@ -0,0 +1,17 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "6.8.3"' + - '"cluster_name" : "six"' + - '"name" : "six-master-0"' + - 'You Know, for Search' diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/values.yaml new file mode 100755 index 000000000..858a5ec58 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/6.x/values.yaml @@ -0,0 +1,4 @@ +--- + +clusterName: "six" +imageTag: "6.8.3" diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/config/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/config/Makefile new file mode 100755 index 000000000..a3f96174b --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/config/Makefile @@ -0,0 +1,19 @@ +default: test +include ../../../helpers/examples.mk + +RELEASE := helm-es-config + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + +secrets: + kubectl delete secret elastic-config-credentials elastic-config-secret elastic-config-slack elastic-config-custom-path || true + kubectl create secret generic elastic-config-credentials --from-literal=password=changeme --from-literal=username=elastic + kubectl create secret generic elastic-config-slack --from-literal=xpack.notification.slack.account.monitoring.secure_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' + kubectl create secret generic elastic-config-secret --from-file=xpack.watcher.encryption_key=./watcher_encryption_key + kubectl create secret generic elastic-config-custom-path --from-literal=slack_url='https://hooks.slack.com/services/asdasdasd/asdasdas/asdasd' --from-literal=thing_i_don_tcare_about=test + +test: secrets install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/config/README.md b/kubernetes/helm_charts/logging/elasticsearch/examples/config/README.md new file mode 100755 index 000000000..d98d836bf --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/config/README.md @@ -0,0 +1,3 @@ +# Config + +An example testing suite for testing some of the optional features of this chart. diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/config/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/config/test/goss.yaml new file mode 100755 index 000000000..848701370 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/config/test/goss.yaml @@ -0,0 +1,26 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":1' + - '"number_of_data_nodes":1' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"cluster_name" : "config"' + - '"name" : "config-master-0"' + - 'You Know, for Search' + +command: + "elasticsearch-keystore list": + exit-status: 0 + stdout: + - keystore.seed + - bootstrap.password + - xpack.notification.slack.account.monitoring.secure_url + - xpack.notification.slack.account.otheraccount.secure_url + - xpack.watcher.encryption_key diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/config/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/config/values.yaml new file mode 100755 index 000000000..ebde4f4d9 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/config/values.yaml @@ -0,0 +1,31 @@ +--- + +clusterName: "config" +replicas: 1 + +extraEnvs: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password + - name: ELASTIC_USERNAME + valueFrom: + secretKeyRef: + name: elastic-credentials + key: username + +# This is just a dummy file to make sure that +# the keystore can be mounted at the same time +# as a custom elasticsearch.yml +esConfig: + elasticsearch.yml: | + path.data: /usr/share/elasticsearch/data + +keystore: + - secretName: elastic-config-secret + - secretName: elastic-config-slack + - secretName: elastic-config-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/config/watcher_encryption_key b/kubernetes/helm_charts/logging/elasticsearch/examples/config/watcher_encryption_key new file mode 100755 index 000000000..b5f907866 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/config/watcher_encryption_key @@ -0,0 +1 @@ +supersecret diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/default/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/default/Makefile new file mode 100755 index 000000000..5f5215c6b --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/default/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-es-default + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) ../../ + +restart: + helm upgrade --set terminationGracePeriod=121 --wait --timeout=600 --install $(RELEASE) ../../ + +test: install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/default/rolling_upgrade.sh b/kubernetes/helm_charts/logging/elasticsearch/examples/default/rolling_upgrade.sh new file mode 100755 index 000000000..c5a2a887b --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/default/rolling_upgrade.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash -x + +kubectl proxy || true & + +make & +PROC_ID=$! + +while kill -0 "$PROC_ID" >/dev/null 2>&1; do + echo "PROCESS IS RUNNING" + if curl --fail 'http://localhost:8001/api/v1/proxy/namespaces/default/services/elasticsearch-master:9200/_search' ; then + echo "cluster is healthy" + else + echo "cluster not healthy!" + exit 1 + fi + sleep 1 +done +echo "PROCESS TERMINATED" +exit 0 diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/default/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/default/test/goss.yaml new file mode 100755 index 000000000..182981df0 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/default/test/goss.yaml @@ -0,0 +1,39 @@ +kernel-param: + vm.max_map_count: + value: '262144' + +http: + http://elasticsearch-master:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.3.2"' + - '"cluster_name" : "elasticsearch"' + - '"name" : "elasticsearch-master-0"' + - 'You Know, for Search' + +file: + /usr/share/elasticsearch/data: + exists: true + mode: "2775" + owner: root + group: elasticsearch + filetype: directory + +mount: + /usr/share/elasticsearch/data: + exists: true + +user: + elasticsearch: + exists: true + uid: 1000 + gid: 1000 diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/Makefile new file mode 100755 index 000000000..398545e64 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/Makefile @@ -0,0 +1,12 @@ +default: test + +RELEASE := helm-es-docker-for-mac + +install: + helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + +test: install + helm test $(RELEASE) + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/values.yaml new file mode 100755 index 000000000..f7deba66b --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/docker-for-mac/values.yaml @@ -0,0 +1,23 @@ +--- +# Permit co-located instances for solitary minikube virtual machines. +antiAffinity: "soft" + +# Shrink default JVM heap. +esJavaOpts: "-Xmx128m -Xms128m" + +# Allocate smaller chunks of memory per pod. +resources: + requests: + cpu: "100m" + memory: "512M" + limits: + cpu: "1000m" + memory: "512M" + +# Request smaller persistent volumes. +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "hostpath" + resources: + requests: + storage: 100M diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/migration/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/Makefile new file mode 100755 index 000000000..3b1dac105 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/Makefile @@ -0,0 +1,10 @@ +PREFIX := helm-es-migration + +data: + helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ + +master: + helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ + +client: + helm upgrade --wait --timeout=600 --install --values ./client.yml $(PREFIX)-client ../../ diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/migration/README.md b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/README.md new file mode 100755 index 000000000..e5f4b1a79 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/README.md @@ -0,0 +1,86 @@ +# Migration Guide from helm/charts + +There are two viable options for migrating from the community Elasticsearch helm chart from the [helm/charts](https://github.com/helm/charts/tree/master/stable/elasticsearch) repo. + +1. Restoring from Snapshot to a fresh cluster +2. Live migration by joining a new cluster to the existing cluster. + +## Restoring from Snapshot + +This is the recommended and preferred option. The downside is that it will involve a period of write downtime during the migration. If you have a way to temporarily stop writes to your cluster then this is the way to go. This is also a lot simpler as it just involves launching a fresh cluster and restoring a snapshot following the [restoring to a different cluster guide](https://www.elastic.co/guide/en/elasticsearch/reference/6.6/modules-snapshots.html#_restoring_to_a_different_cluster). + +## Live migration + +If restoring from a snapshot is not possible due to the write downtime then a live migration is also possible. It is very important to first test this in a testing environment to make sure you are comfortable with the process and fully understand what is happening. + +This process will involve joining a new set of master, data and client nodes to an existing cluster that has been deployed using the [helm/charts](https://github.com/helm/charts/tree/master/stable/elasticsearch) community chart. Nodes will then be replaced one by one in a controlled fashion to decommission the old cluster. + +This example will be using the default values for the existing helm/charts release and for the elastic helm-charts release. If you have changed any of the default values then you will need to first make sure that your values are configured in a compatible way before starting the migration. + +The process will involve a re-sync and a rolling restart of all of your data nodes. Therefore it is important to disable shard allocation and perform a synced flush like you normally would during any other rolling upgrade. See the [rolling upgrades guide](https://www.elastic.co/guide/en/elasticsearch/reference/6.6/rolling-upgrades.html) for more information. + +* The default image for this chart is `docker.elastic.co/elasticsearch/elasticsearch` which contains the default distribution of Elasticsearch with a [basic license](https://www.elastic.co/subscriptions). Make sure to update the `image` and `imageTag` values to the correct Docker image and Elasticsearch version that you currently have deployed. +* Convert your current helm/charts configuration into something that is compatible with this chart. +* Take a fresh snapshot of your cluster. If something goes wrong you want to be able to restore your data no matter what. +* Check that your clusters health is green. If not abort and make sure your cluster is healthy before continuing. + ``` + curl localhost:9200/_cluster/health + ``` +* Deploy new data nodes which will join the existing cluster. Take a look at the configuration in [data.yml](./data.yml) + ``` + make data + ``` +* Check that the new nodes have joined the cluster (run this and any other curl commands from within one of your pods). + ``` + curl localhost:9200/_cat/nodes + ``` +* Check that your cluster is still green. If so we can now start to scale down the existing data nodes. Assuming you have the default amount of data nodes (2) we now want to scale it down to 1. + ``` + kubectl scale statefulsets my-release-elasticsearch-data --replicas=1 + ``` +* Wait for your cluster to become green again + ``` + watch 'curl -s localhost:9200/_cluster/health' + ``` +* Once the cluster is green we can scale down again. + ``` + kubectl scale statefulsets my-release-elasticsearch-data --replicas=0 + ``` +* Wait for the cluster to be green again. +* OK. We now have all data nodes running in the new cluster. Time to replace the masters by firstly scaling down the masters from 3 to 2. Between each step make sure to wait for the cluster to become green again, and check with `curl localhost:9200/_cat/nodes` that you see the correct amount of master nodes. During this process we will always make sure to keep at least 2 master nodes as to not lose quorum. + ``` + kubectl scale statefulsets my-release-elasticsearch-master --replicas=2 + ``` +* Now deploy a single new master so that we have 3 masters again. See [master.yml](./master.yml) for the configuration. + ``` + make master + ``` +* Scale down old masters to 1 + ``` + kubectl scale statefulsets my-release-elasticsearch-master --replicas=1 + ``` +* Edit the masters in [masters.yml](./masters.yml) to 2 and redeploy + ``` + make master + ``` +* Scale down the old masters to 0 + ``` + kubectl scale statefulsets my-release-elasticsearch-master --replicas=0 + ``` +* Edit the [masters.yml](./masters.yml) to have 3 replicas and remove the `discovery.zen.ping.unicast.hosts` entry from `extraEnvs` then redeploy the masters. This will make sure all 3 masters are running in the new cluster and are pointing at each other for discovery. + ``` + make master + ``` +* Remove the `discovery.zen.ping.unicast.hosts` entry from `extraEnvs` then redeploy the data nodes to make sure they are pointing at the new masters. + ``` + make data + ``` +* Deploy the client nodes + ``` + make client + ``` +* Update any processes that are talking to the existing client nodes and point them to the new client nodes. Once this is done you can scale down the old client nodes + ``` + kubectl scale deployment my-release-elasticsearch-client --replicas=0 + ``` +* The migration should now be complete. After verifying that everything is working correctly you can cleanup leftover resources from your old cluster. diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/migration/client.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/client.yml new file mode 100755 index 000000000..30ee700e3 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/client.yml @@ -0,0 +1,23 @@ +--- + +replicas: 2 + +clusterName: "elasticsearch" +nodeGroup: "client" + +esMajorVersion: 6 + +roles: + master: "false" + ingest: "false" + data: "false" + +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 1Gi # Currently needed till pvcs are made optional + +persistence: + enabled: false diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/migration/data.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/data.yml new file mode 100755 index 000000000..eedcbb063 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/data.yml @@ -0,0 +1,17 @@ +--- + +replicas: 2 + +esMajorVersion: 6 + +extraEnvs: + - name: discovery.zen.ping.unicast.hosts + value: "my-release-elasticsearch-discovery" + +clusterName: "elasticsearch" +nodeGroup: "data" + +roles: + master: "false" + ingest: "false" + data: "true" diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/migration/master.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/master.yml new file mode 100755 index 000000000..3e3a2f172 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/migration/master.yml @@ -0,0 +1,26 @@ +--- + +# Temporarily set to 3 so we can scale up/down the old a new cluster +# one at a time whilst always keeping 3 masters running +replicas: 1 + +esMajorVersion: 6 + +extraEnvs: + - name: discovery.zen.ping.unicast.hosts + value: "my-release-elasticsearch-discovery" + +clusterName: "elasticsearch" +nodeGroup: "master" + +roles: + master: "true" + ingest: "false" + data: "false" + +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 4Gi diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/Makefile new file mode 100755 index 000000000..97109ce8c --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/Makefile @@ -0,0 +1,12 @@ +default: test + +RELEASE := helm-es-minikube + +install: + helm upgrade --wait --timeout=900 --install --values values.yaml $(RELEASE) ../../ + +test: install + helm test $(RELEASE) + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/values.yaml new file mode 100755 index 000000000..ccceb3a25 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/minikube/values.yaml @@ -0,0 +1,23 @@ +--- +# Permit co-located instances for solitary minikube virtual machines. +antiAffinity: "soft" + +# Shrink default JVM heap. +esJavaOpts: "-Xmx128m -Xms128m" + +# Allocate smaller chunks of memory per pod. +resources: + requests: + cpu: "100m" + memory: "512M" + limits: + cpu: "1000m" + memory: "512M" + +# Request smaller persistent volumes. +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "standard" + resources: + requests: + storage: 100M diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/multi/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/Makefile new file mode 100755 index 000000000..836ec2e0f --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/Makefile @@ -0,0 +1,16 @@ +default: test + +include ../../../helpers/examples.mk + +PREFIX := helm-es-multi +RELEASE := helm-es-multi-master + +install: + helm upgrade --wait --timeout=600 --install --values ./master.yml $(PREFIX)-master ../../ + helm upgrade --wait --timeout=600 --install --values ./data.yml $(PREFIX)-data ../../ + +test: install goss + +purge: + helm del --purge $(PREFIX)-master + helm del --purge $(PREFIX)-data diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/multi/data.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/data.yml new file mode 100755 index 000000000..ecc689337 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/data.yml @@ -0,0 +1,9 @@ +--- + +clusterName: "multi" +nodeGroup: "data" + +roles: + master: "false" + ingest: "true" + data: "true" diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/multi/master.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/master.yml new file mode 100755 index 000000000..2ca4cca8e --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/master.yml @@ -0,0 +1,9 @@ +--- + +clusterName: "multi" +nodeGroup: "master" + +roles: + master: "true" + ingest: "false" + data: "false" diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/multi/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/test/goss.yaml new file mode 100755 index 000000000..18cb25063 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/multi/test/goss.yaml @@ -0,0 +1,9 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"cluster_name":"multi"' + - '"number_of_nodes":6' + - '"number_of_data_nodes":3' diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/Makefile new file mode 100755 index 000000000..6e495916d --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/Makefile @@ -0,0 +1,15 @@ +default: test +include ../../../helpers/examples.mk + +RELEASE := elasticsearch + +template: + helm template --values ./values.yaml ../../ + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + +test: install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/test/goss.yaml new file mode 100755 index 000000000..275754bb3 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/test/goss.yaml @@ -0,0 +1,17 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.3.0"' + - '"cluster_name" : "elasticsearch"' + - '"name" : "elasticsearch-master-0"' + - 'You Know, for Search' diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/values.yaml new file mode 100755 index 000000000..7f5cd8494 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/openshift/values.yaml @@ -0,0 +1,10 @@ +--- + +securityContext: + runAsUser: null + +podSecurityContext: + fsGroup: null + +sysctlInitContainer: + enabled: false diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/oss/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/Makefile new file mode 100755 index 000000000..e274659c6 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/Makefile @@ -0,0 +1,12 @@ +default: test +include ../../../helpers/examples.mk + +RELEASE := helm-es-oss + +install: + helm upgrade --wait --timeout=600 --install $(RELEASE) --values ./values.yaml ../../ + +test: install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/oss/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/test/goss.yaml new file mode 100755 index 000000000..21e268b0d --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/test/goss.yaml @@ -0,0 +1,17 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.3.2"' + - '"cluster_name" : "oss"' + - '"name" : "oss-master-0"' + - 'You Know, for Search' diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/oss/values.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/values.yaml new file mode 100755 index 000000000..adcb7df3e --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/oss/values.yaml @@ -0,0 +1,4 @@ +--- + +clusterName: "oss" +image: "docker.elastic.co/elasticsearch/elasticsearch-oss" diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/security/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/security/Makefile new file mode 100755 index 000000000..1c7e7c28d --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/security/Makefile @@ -0,0 +1,31 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-es-security + +install: + helm upgrade --wait --timeout=600 --install --values ./security.yml $(RELEASE) ../../ + +purge: + kubectl delete secrets elastic-credentials elastic-certificates elastic-certificate-pem || true + helm del --purge $(RELEASE) + +test: secrets install goss + +secrets: + docker rm -f elastic-helm-charts-certs || true + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 || true + password=$$([ ! -z "$$ELASTIC_PASSWORD" ] && echo $$ELASTIC_PASSWORD || echo $$(docker run --rm docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) /bin/sh -c "< /dev/urandom tr -cd '[:alnum:]' | head -c20")) && \ + docker run --name elastic-helm-charts-certs -i -w /app \ + docker.elastic.co/elasticsearch/elasticsearch:$(STACK_VERSION) \ + /bin/sh -c " \ + elasticsearch-certutil ca --out /app/elastic-stack-ca.p12 --pass '' && \ + elasticsearch-certutil cert --name security-master --dns security-master --ca /app/elastic-stack-ca.p12 --pass '' --ca-pass '' --out /app/elastic-certificates.p12" && \ + docker cp elastic-helm-charts-certs:/app/elastic-certificates.p12 ./ && \ + docker rm -f elastic-helm-charts-certs && \ + openssl pkcs12 -nodes -passin pass:'' -in elastic-certificates.p12 -out elastic-certificate.pem && \ + kubectl create secret generic elastic-certificates --from-file=elastic-certificates.p12 && \ + kubectl create secret generic elastic-certificate-pem --from-file=elastic-certificate.pem && \ + kubectl create secret generic elastic-credentials --from-literal=password=$$password --from-literal=username=elastic && \ + rm -f elastic-certificates.p12 elastic-certificate.pem elastic-stack-ca.p12 diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/security/security.yml b/kubernetes/helm_charts/logging/elasticsearch/examples/security/security.yml new file mode 100755 index 000000000..04d932caa --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/security/security.yml @@ -0,0 +1,38 @@ +--- +clusterName: "security" +nodeGroup: "master" + +roles: + master: "true" + ingest: "true" + data: "true" + +protocol: https + +esConfig: + elasticsearch.yml: | + xpack.security.enabled: true + xpack.security.transport.ssl.enabled: true + xpack.security.transport.ssl.verification_mode: certificate + xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 + xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 + xpack.security.http.ssl.enabled: true + xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 + xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12 + +extraEnvs: + - name: ELASTIC_PASSWORD + valueFrom: + secretKeyRef: + name: elastic-credentials + key: password + - name: ELASTIC_USERNAME + valueFrom: + secretKeyRef: + name: elastic-credentials + key: username + +secretMounts: + - name: elastic-certificates + secretName: elastic-certificates + path: /usr/share/elasticsearch/config/certs diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/security/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/security/test/goss.yaml new file mode 100755 index 000000000..c6d4b987b --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/security/test/goss.yaml @@ -0,0 +1,45 @@ +http: + https://security-master:9200/_cluster/health: + status: 200 + timeout: 2000 + allow-insecure: true + username: '{{ .Env.ELASTIC_USERNAME }}' + password: '{{ .Env.ELASTIC_PASSWORD }}' + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + https://localhost:9200/: + status: 200 + timeout: 2000 + allow-insecure: true + username: '{{ .Env.ELASTIC_USERNAME }}' + password: '{{ .Env.ELASTIC_PASSWORD }}' + body: + - '"cluster_name" : "security"' + - '"name" : "security-master-0"' + - 'You Know, for Search' + + https://localhost:9200/_xpack/license: + status: 200 + timeout: 2000 + allow-insecure: true + username: '{{ .Env.ELASTIC_USERNAME }}' + password: '{{ .Env.ELASTIC_PASSWORD }}' + body: + - 'active' + - 'basic' + +file: + /usr/share/elasticsearch/config/elasticsearch.yml: + exists: true + contains: + - 'xpack.security.enabled: true' + - 'xpack.security.transport.ssl.enabled: true' + - 'xpack.security.transport.ssl.verification_mode: certificate' + - 'xpack.security.transport.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' + - 'xpack.security.transport.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' + - 'xpack.security.http.ssl.enabled: true' + - 'xpack.security.http.ssl.truststore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' + - 'xpack.security.http.ssl.keystore.path: /usr/share/elasticsearch/config/certs/elastic-certificates.p12' diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/Makefile b/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/Makefile new file mode 100755 index 000000000..9e1e6fd5e --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/Makefile @@ -0,0 +1,25 @@ +default: test + +include ../../../helpers/examples.mk + +RELEASE := helm-es-upgrade + +# Right now the version is hardcoded because helm install will ignore +# anything with an alpha tag when trying to install the latest release +# This hardcoding can be removed once we drop the alpha tag +# The "--set terminationGracePeriod=121" always makes sure that a rolling +# upgrade is forced for this test +install: + helm repo add elastic https://helm.elastic.co && \ + helm upgrade --wait --timeout=600 --install $(RELEASE) elastic/elasticsearch --version 7.0.0-alpha1 --set clusterName=upgrade ; \ + kubectl rollout status sts/upgrade-master --timeout=600s + helm upgrade --wait --timeout=600 --set terminationGracePeriod=121 --install $(RELEASE) ../../ --set clusterName=upgrade ; \ + kubectl rollout status sts/upgrade-master --timeout=600s + +init: + helm init --client-only + +test: init install goss + +purge: + helm del --purge $(RELEASE) diff --git a/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/test/goss.yaml b/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/test/goss.yaml new file mode 100755 index 000000000..5e20f7fea --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/examples/upgrade/test/goss.yaml @@ -0,0 +1,17 @@ +http: + http://localhost:9200/_cluster/health: + status: 200 + timeout: 2000 + body: + - 'green' + - '"number_of_nodes":3' + - '"number_of_data_nodes":3' + + http://localhost:9200: + status: 200 + timeout: 2000 + body: + - '"number" : "7.3.2"' + - '"cluster_name" : "upgrade"' + - '"name" : "upgrade-master-0"' + - 'You Know, for Search' diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/NOTES.txt b/kubernetes/helm_charts/logging/elasticsearch/templates/NOTES.txt new file mode 100755 index 000000000..3600c6b55 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/NOTES.txt @@ -0,0 +1,4 @@ +1. Watch all cluster members come up. + $ kubectl get pods --namespace={{ .Release.Namespace }} -l app={{ template "uname" . }} -w +2. Test cluster health using Helm test. + $ helm test {{ .Release.Name }} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/_helpers.tpl b/kubernetes/helm_charts/logging/elasticsearch/templates/_helpers.tpl new file mode 100755 index 000000000..ccfd8c338 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/_helpers.tpl @@ -0,0 +1,71 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "uname" -}} +{{ .Values.clusterName }}-{{ .Values.nodeGroup }} +{{- end -}} + +{{- define "masterService" -}} +{{- if empty .Values.masterService -}} +{{ .Values.clusterName }}-master +{{- else -}} +{{ .Values.masterService }} +{{- end -}} +{{- end -}} + +{{- define "endpoints" -}} +{{- $replicas := .Values.replicas | int }} +{{- $uname := printf "%s-%s" .Values.clusterName .Values.nodeGroup }} + {{- range $i, $e := untilStep 0 $replicas 1 -}} +{{ $uname }}-{{ $i }}, + {{- end -}} +{{- end -}} + +{{- define "esMajorVersion" -}} +{{- if .Values.esMajorVersion -}} +{{ .Values.esMajorVersion }} +{{- else -}} +{{- $version := int (index (.Values.imageTag | splitList ".") 0) -}} + {{- if and (contains "docker.elastic.co/elasticsearch/elasticsearch" .Values.image) (not (eq $version 0)) -}} +{{ $version }} + {{- else -}} +7 + {{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "elasticsearch.statefulset.apiVersion" -}} +{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "apps/v1beta2" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "elasticsearch.ingress.apiVersion" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/configmap.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/configmap.yaml new file mode 100755 index 000000000..6d252ae7e --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/configmap.yaml @@ -0,0 +1,17 @@ +{{- if .Values.esConfig }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "uname" . }}-config + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" +data: +{{- range $path, $config := .Values.esConfig }} + {{ $path }}: | +{{ $config | indent 4 -}} +{{- end -}} +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/ingress.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/ingress.yaml new file mode 100755 index 000000000..71891f9b6 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/ingress.yaml @@ -0,0 +1,39 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "uname" . -}} +{{- $servicePort := .Values.httpPort -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: {{ template "elasticsearch.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + namespace: {{ .Values.namespace }} + labels: + app: {{ .Chart.Name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.ingress.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} +spec: +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . }} + http: + paths: + - path: {{ $ingressPath }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/poddisruptionbudget.yaml new file mode 100755 index 000000000..21b63b345 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/poddisruptionbudget.yaml @@ -0,0 +1,13 @@ +--- +{{- if .Values.maxUnavailable }} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: "{{ template "uname" . }}-pdb" + namespace: {{ .Values.namespace }} +spec: + maxUnavailable: {{ .Values.maxUnavailable }} + selector: + matchLabels: + app: "{{ template "uname" . }}" +{{- end }} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/podsecuritypolicy.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/podsecuritypolicy.yaml new file mode 100755 index 000000000..aa8aa7090 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/podsecuritypolicy.yaml @@ -0,0 +1,15 @@ +{{- if .Values.podSecurityPolicy.create -}} +{{- $fullName := include "uname" . -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ default $fullName .Values.podSecurityPolicy.name | quote }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ $fullName | quote }} +spec: +{{ toYaml .Values.podSecurityPolicy.spec | indent 2 }} +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/role.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/role.yaml new file mode 100755 index 000000000..3259959f6 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/role.yaml @@ -0,0 +1,26 @@ +{{- if .Values.rbac.create -}} +{{- $fullName := include "uname" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $fullName | quote }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ $fullName | quote }} +rules: + - apiGroups: + - extensions + resources: + - podsecuritypolicies + resourceNames: + {{- if eq .Values.podSecurityPolicy.name "" }} + - {{ $fullName | quote }} + {{- else }} + - {{ .Values.podSecurityPolicy.name | quote }} + {{- end }} + verbs: + - use +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/rolebinding.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/rolebinding.yaml new file mode 100755 index 000000000..953c8cb3a --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/rolebinding.yaml @@ -0,0 +1,25 @@ +{{- if .Values.rbac.create -}} +{{- $fullName := include "uname" . -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $fullName | quote }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ $fullName | quote }} +subjects: + - kind: ServiceAccount + {{- if eq .Values.rbac.serviceAccountName "" }} + name: {{ $fullName | quote }} + {{- else }} + name: {{ .Values.rbac.serviceAccountName | quote }} + {{- end }} + namespace: {{ .Values.namespace }} +roleRef: + kind: Role + name: {{ $fullName | quote }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/service.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/service.yaml new file mode 100755 index 000000000..83bfc9a90 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/service.yaml @@ -0,0 +1,54 @@ +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ template "uname" . }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" + annotations: +{{ toYaml .Values.service.annotations | indent 4 }} +spec: + type: {{ .Values.service.type }} + selector: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" + ports: + - name: {{ .Values.service.httpPortName | default "http" }} + protocol: TCP + port: {{ .Values.httpPort }} +{{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} +{{- end }} + - name: {{ .Values.service.transportPortName | default "transport" }} + protocol: TCP + port: {{ .Values.transportPort }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ template "uname" . }}-headless + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" +spec: + clusterIP: None # This is needed for statefulset hostnames like elasticsearch-0 to resolve + # Create endpoints also if the related pod isn't ready + publishNotReadyAddresses: true + selector: + app: "{{ template "uname" . }}" + ports: + - name: {{ .Values.service.httpPortName | default "http" }} + port: {{ .Values.httpPort }} + - name: {{ .Values.service.transportPortName | default "transport" }} + port: {{ .Values.transportPort }} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/serviceaccount.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/serviceaccount.yaml new file mode 100755 index 000000000..dc38334b9 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/serviceaccount.yaml @@ -0,0 +1,17 @@ +{{- if .Values.rbac.create -}} +{{- $fullName := include "uname" . -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + {{- if eq .Values.rbac.serviceAccountName "" }} + name: {{ $fullName | quote }} + {{- else }} + name: {{ .Values.rbac.serviceAccountName | quote }} + {{- end }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + app: {{ $fullName | quote }} +{{- end -}} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/statefulset.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/statefulset.yaml new file mode 100755 index 000000000..59729e334 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/statefulset.yaml @@ -0,0 +1,345 @@ +--- +apiVersion: {{ template "elasticsearch.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ template "uname" . }} + namespace: {{ .Values.namespace }} + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" + {{- range $key, $value := .Values.labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + annotations: + esMajorVersion: "{{ include "esMajorVersion" . }}" +spec: + serviceName: {{ template "uname" . }}-headless + selector: + matchLabels: + app: "{{ template "uname" . }}" + replicas: {{ default .Values.replicas }} + podManagementPolicy: {{ .Values.podManagementPolicy }} + updateStrategy: + type: {{ .Values.updateStrategy }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ template "uname" . }} + {{- with .Values.persistence.annotations }} + annotations: +{{ toYaml . | indent 8 }} + {{- end }} + spec: +{{ toYaml .Values.volumeClaimTemplate | indent 6 }} + {{- end }} + template: + metadata: + name: "{{ template "uname" . }}" + labels: + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} + chart: "{{ .Chart.Name }}" + app: "{{ template "uname" . }}" + annotations: + {{- range $key, $value := .Values.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{/* This forces a restart if the configmap has changed */}} + {{- if .Values.esConfig }} + configchecksum: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum | trunc 63 }} + {{- end }} + spec: + {{- if .Values.schedulerName }} + schedulerName: "{{ .Values.schedulerName }}" + {{- end }} + securityContext: +{{ toYaml .Values.podSecurityContext | indent 8 }} + {{- if .Values.fsGroup }} + fsGroup: {{ .Values.fsGroup }} # Deprecated value, please use .Values.podSecurityContext.fsGroup + {{- end }} + {{- if .Values.rbac.create }} + serviceAccountName: "{{ template "uname" . }}" + {{- else if not (eq .Values.rbac.serviceAccountName "") }} + serviceAccountName: {{ .Values.rbac.serviceAccountName | quote }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: +{{ toYaml . | indent 6 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- if or (eq .Values.antiAffinity "hard") (eq .Values.antiAffinity "soft") .Values.nodeAffinity }} + {{- if .Values.priorityClassName }} + priorityClassName: {{ .Values.priorityClassName }} + {{- end }} + affinity: + {{- end }} + {{- if eq .Values.antiAffinity "hard" }} + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - "{{ template "uname" .}}" + topologyKey: {{ .Values.antiAffinityTopologyKey }} + {{- else if eq .Values.antiAffinity "soft" }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + topologyKey: {{ .Values.antiAffinityTopologyKey }} + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - "{{ template "uname" . }}" + {{- end }} + {{- with .Values.nodeAffinity }} + nodeAffinity: +{{ toYaml . | indent 10 }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriod }} + volumes: + {{- range .Values.secretMounts }} + - name: {{ .name }} + secret: + secretName: {{ .secretName }} + {{- end }} + {{- if .Values.esConfig }} + - name: esconfig + configMap: + name: {{ template "uname" . }}-config + {{- end }} +{{- if .Values.keystore }} + - name: keystore + emptyDir: {} + {{- range .Values.keystore }} + - name: keystore-{{ .secretName }} + secret: {{ toYaml . | nindent 12 }} + {{- end }} +{{ end }} + {{- if .Values.extraVolumes }} +{{ tpl .Values.extraVolumes . | indent 8 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: +{{ toYaml .Values.imagePullSecrets | indent 8 }} + {{- end }} + initContainers: + {{- if .Values.sysctlInitContainer.enabled }} + - name: configure-sysctl + securityContext: + runAsUser: 0 + privileged: true + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + command: ["sysctl", "-w", "vm.max_map_count={{ .Values.sysctlVmMaxMapCount}}"] + resources: +{{ toYaml .Values.initResources | indent 10 }} + {{- end }} +{{ if .Values.keystore }} + - name: keystore + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + command: + - sh + - -c + - | + #!/usr/bin/env bash + set -euo pipefail + + elasticsearch-keystore create + + for i in /tmp/keystoreSecrets/*/*; do + key=$(basename $i) + echo "Adding file $i to keystore key $key" + elasticsearch-keystore add-file "$key" "$i" + done + + # Add the bootstrap password since otherwise the Elasticsearch entrypoint tries to do this on startup + [ ! -z ${ELASTIC_PASSWORD+x} ] && echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x bootstrap.password + + cp -a /usr/share/elasticsearch/config/elasticsearch.keystore /tmp/keystore/ + env: {{ toYaml .Values.extraEnvs | nindent 10 }} + resources: {{ toYaml .Values.initResources | nindent 10 }} + volumeMounts: + - name: keystore + mountPath: /tmp/keystore + {{- range .Values.keystore }} + - name: keystore-{{ .secretName }} + mountPath: /tmp/keystoreSecrets/{{ .secretName }} + {{- end }} +{{ end }} + {{- if .Values.extraInitContainers }} +{{ tpl .Values.extraInitContainers . | indent 6 }} + {{- end }} + containers: + - name: "{{ template "name" . }}" + securityContext: +{{ toYaml .Values.securityContext | indent 10 }} + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + readinessProbe: +{{ toYaml .Values.readinessProbe | indent 10 }} + exec: + command: + - sh + - -c + - | + #!/usr/bin/env bash -e + # If the node is starting up wait for the cluster to be ready (request params: '{{ .Values.clusterHealthCheckParams }}' ) + # Once it has started only check that the node itself is responding + START_FILE=/tmp/.es_start_file + + http () { + local path="${1}" + if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then + BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" + else + BASIC_AUTH='' + fi + curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://127.0.0.1:{{ .Values.httpPort }}${path} + } + + if [ -f "${START_FILE}" ]; then + echo 'Elasticsearch is already running, lets check the node is healthy' + http "/" + else + echo 'Waiting for elasticsearch cluster to become cluster to be ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' + if http "/_cluster/health?{{ .Values.clusterHealthCheckParams }}" ; then + touch ${START_FILE} + exit 0 + else + echo 'Cluster is not yet ready (request params: "{{ .Values.clusterHealthCheckParams }}" )' + exit 1 + fi + fi + ports: + - name: http + containerPort: {{ .Values.httpPort }} + - name: transport + containerPort: {{ .Values.transportPort }} + resources: +{{ toYaml .Values.resources | indent 10 }} + env: + - name: node.name + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- if eq .Values.roles.master "true" }} + {{- if ge (int (include "esMajorVersion" .)) 7 }} + - name: cluster.initial_master_nodes + value: "{{ template "endpoints" . }}" + {{- else }} + - name: discovery.zen.minimum_master_nodes + value: "{{ .Values.minimumMasterNodes }}" + {{- end }} + {{- end }} + {{- if lt (int (include "esMajorVersion" .)) 7 }} + - name: discovery.zen.ping.unicast.hosts + value: "{{ template "masterService" . }}-headless" + {{- else }} + - name: discovery.seed_hosts + value: "{{ template "masterService" . }}-headless" + {{- end }} + - name: cluster.name + value: "{{ .Values.clusterName }}" + - name: network.host + value: "{{ .Values.networkHost }}" + - name: ES_JAVA_OPTS + value: "{{ .Values.esJavaOpts }}" + {{- range $role, $enabled := .Values.roles }} + - name: node.{{ $role }} + value: "{{ $enabled }}" + {{- end }} +{{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 10 }} +{{- end }} + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: "{{ template "uname" . }}" + mountPath: /usr/share/elasticsearch/data + {{- end }} +{{ if .Values.keystore }} + - name: keystore + mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore + subPath: elasticsearch.keystore +{{ end }} + {{- range .Values.secretMounts }} + - name: {{ .name }} + mountPath: {{ .path }} + {{- if .subPath }} + subPath: {{ .subPath }} + {{- end }} + {{- end }} + {{- range $path, $config := .Values.esConfig }} + - name: esconfig + mountPath: /usr/share/elasticsearch/config/{{ $path }} + subPath: {{ $path }} + {{- end -}} + {{- if .Values.extraVolumeMounts }} +{{ tpl .Values.extraVolumeMounts . | indent 10 }} + {{- end }} + {{- if .Values.masterTerminationFix }} + {{- if eq .Values.roles.master "true" }} + # This sidecar will prevent slow master re-election + # https://github.com/elastic/helm-charts/issues/63 + - name: elasticsearch-master-graceful-termination-handler + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + imagePullPolicy: "{{ .Values.imagePullPolicy }}" + command: + - "sh" + - -c + - | + #!/usr/bin/env bash + set -eo pipefail + + http () { + local path="${1}" + if [ -n "${ELASTIC_USERNAME}" ] && [ -n "${ELASTIC_PASSWORD}" ]; then + BASIC_AUTH="-u ${ELASTIC_USERNAME}:${ELASTIC_PASSWORD}" + else + BASIC_AUTH='' + fi + curl -XGET -s -k --fail ${BASIC_AUTH} {{ .Values.protocol }}://{{ template "masterService" . }}:{{ .Values.httpPort }}${path} + } + + cleanup () { + while true ; do + local master="$(http "/_cat/master?h=node" || echo "")" + if [[ $master == "{{ template "masterService" . }}"* && $master != "${NODE_NAME}" ]]; then + echo "This node is not master." + break + fi + echo "This node is still master, waiting gracefully for it to step down" + sleep 1 + done + + exit 0 + } + + trap cleanup SIGTERM + + sleep infinity & + wait $! + resources: +{{ toYaml .Values.sidecarResources | indent 10 }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{- if .Values.extraEnvs }} +{{ toYaml .Values.extraEnvs | indent 10 }} + {{- end }} + {{- end }} + {{- end }} +{{- if .Values.lifecycle }} + lifecycle: +{{ toYaml .Values.lifecycle | indent 10 }} +{{- end }} diff --git a/kubernetes/helm_charts/logging/elasticsearch/templates/test/test-elasticsearch-health.yaml b/kubernetes/helm_charts/logging/elasticsearch/templates/test/test-elasticsearch-health.yaml new file mode 100755 index 000000000..14541156a --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/templates/test/test-elasticsearch-health.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ .Release.Name }}-{{ randAlpha 5 | lower }}-test" + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: "{{ .Release.Name }}-{{ randAlpha 5 | lower }}-test" + image: "{{ .Values.image }}:{{ .Values.imageTag }}" + command: + - "sh" + - "-c" + - | + #!/usr/bin/env bash -e + curl -XGET --fail '{{ template "uname" . }}:{{ .Values.httpPort }}/_cluster/health?{{ .Values.clusterHealthCheckParams }}' + restartPolicy: Never diff --git a/kubernetes/helm_charts/logging/elasticsearch/tests/elasticsearch_test.py b/kubernetes/helm_charts/logging/elasticsearch/tests/elasticsearch_test.py new file mode 100755 index 000000000..4517c9fc7 --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/tests/elasticsearch_test.py @@ -0,0 +1,971 @@ +import os +import sys +sys.path.insert(1, os.path.join(sys.path[0], '../../helpers')) +from helpers import helm_template +import yaml + +clusterName = 'elasticsearch' +nodeGroup = 'master' +uname = clusterName + '-' + nodeGroup + + +def test_defaults(): + config = ''' + ''' + + r = helm_template(config) + + # Statefulset + assert r['statefulset'][uname]['spec']['replicas'] == 3 + assert r['statefulset'][uname]['spec']['updateStrategy'] == { + 'type': 'RollingUpdate'} + assert r['statefulset'][uname]['spec']['podManagementPolicy'] == 'Parallel' + assert r['statefulset'][uname]['spec']['serviceName'] == uname + '-headless' + assert r['statefulset'][uname]['spec']['template']['spec']['affinity']['podAntiAffinity']['requiredDuringSchedulingIgnoredDuringExecution'][0] == \ + { + "labelSelector": { + "matchExpressions": [ + { + "key": "app", + "operator": "In", + "values": [ + uname + ] + } + ] + }, + "topologyKey": "kubernetes.io/hostname" + } + + # Default environment variables + env_vars = [ + { + 'name': 'node.name', + 'valueFrom': { + 'fieldRef': { + 'fieldPath': 'metadata.name' + } + } + }, + { + 'name': 'cluster.initial_master_nodes', + 'value': uname + '-0,' + + uname + '-1,' + + uname + '-2,' + }, + { + 'name': 'discovery.seed_hosts', + 'value': uname + '-headless' + + }, + { + 'name': 'network.host', + 'value': '0.0.0.0' + }, + { + 'name': 'cluster.name', + 'value': clusterName + }, + { + 'name': 'ES_JAVA_OPTS', + 'value': '-Xmx1g -Xms1g' + }, + { + 'name': 'node.master', + 'value': 'true' + }, + { + 'name': 'node.data', + 'value': 'true' + }, + { + 'name': 'node.ingest', + 'value': 'true' + }, + ] + + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + for env in env_vars: + assert env in c['env'] + + # Image + assert c['image'].startswith( + 'docker.elastic.co/elasticsearch/elasticsearch:') + assert c['imagePullPolicy'] == 'IfNotPresent' + assert c['name'] == 'elasticsearch' + + # Ports + assert c['ports'][0] == {'name': 'http', 'containerPort': 9200} + assert c['ports'][1] == {'name': 'transport', 'containerPort': 9300} + + # Health checks + assert c['readinessProbe']['failureThreshold'] == 3 + assert c['readinessProbe']['initialDelaySeconds'] == 10 + assert c['readinessProbe']['periodSeconds'] == 10 + assert c['readinessProbe']['successThreshold'] == 3 + assert c['readinessProbe']['timeoutSeconds'] == 5 + + assert 'curl' in c['readinessProbe']['exec']['command'][-1] + assert 'http://127.0.0.1:9200' in c['readinessProbe']['exec']['command'][-1] + + # Resources + assert c['resources'] == { + 'requests': { + 'cpu': '100m', + 'memory': '2Gi' + }, + 'limits': { + 'cpu': '1000m', + 'memory': '2Gi' + } + } + + # Mounts + assert c['volumeMounts'][0]['mountPath'] == '/usr/share/elasticsearch/data' + assert c['volumeMounts'][0]['name'] == uname + + v = r['statefulset'][uname]['spec']['volumeClaimTemplates'][0] + assert v['metadata']['name'] == uname + assert v['spec']['accessModes'] == ['ReadWriteOnce'] + assert v['spec']['resources']['requests']['storage'] == '30Gi' + + # Init container + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][0] + assert i['name'] == 'configure-sysctl' + assert i['command'] == ['sysctl', '-w', 'vm.max_map_count=262144'] + assert i['image'].startswith( + 'docker.elastic.co/elasticsearch/elasticsearch:') + assert i['securityContext'] == { + 'privileged': True, + 'runAsUser': 0 + } + + # Other + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext'] == { + 'fsGroup': 1000, + 'runAsUser': 1000 + } + assert r['statefulset'][uname]['spec']['template']['spec']['terminationGracePeriodSeconds'] == 120 + + # Pod disruption budget + assert r['poddisruptionbudget'][uname + + '-pdb']['spec']['maxUnavailable'] == 1 + + # Service + s = r['service'][uname] + assert s['metadata']['name'] == uname + assert s['metadata']['annotations'] == {} + assert s['spec']['type'] == 'ClusterIP' + assert len(s['spec']['ports']) == 2 + assert s['spec']['ports'][0] == { + 'name': 'http', 'port': 9200, 'protocol': 'TCP'} + assert s['spec']['ports'][1] == { + 'name': 'transport', 'port': 9300, 'protocol': 'TCP'} + + # Headless Service + h = r['service'][uname + '-headless'] + assert h['spec']['clusterIP'] == 'None' + assert h['spec']['publishNotReadyAddresses'] == True + assert h['spec']['ports'][0]['name'] == 'http' + assert h['spec']['ports'][0]['port'] == 9200 + assert h['spec']['ports'][1]['name'] == 'transport' + assert h['spec']['ports'][1]['port'] == 9300 + + # Empty customizable defaults + assert 'imagePullSecrets' not in r['statefulset'][uname]['spec']['template']['spec'] + assert 'tolerations' not in r['statefulset'][uname]['spec']['template']['spec'] + assert 'nodeSelector' not in r['statefulset'][uname]['spec']['template']['spec'] + assert 'ingress' not in r + + +def test_increasing_the_replicas(): + config = ''' +replicas: 5 +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['replicas'] == 5 + + +def test_disabling_pod_disruption_budget(): + config = ''' +maxUnavailable: false +''' + r = helm_template(config) + assert 'poddisruptionbudget' not in r + + +def test_overriding_the_image_and_tag(): + config = ''' +image: customImage +imageTag: 6.2.4 +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['image'] == 'customImage:6.2.4' + + +def test_set_discovery_hosts_to_custom_master_service(): + config = ''' +esMajorVersion: 6 +masterService: "elasticsearch-custommaster" +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + assert {'name': 'discovery.zen.ping.unicast.hosts', + 'value': 'elasticsearch-custommaster-headless'} in env + + +def test_set_master_service_to_default_nodegroup_name_if_not_set(): + config = ''' +esMajorVersion: 6 +nodeGroup: "data" +''' + r = helm_template(config) + env = r['statefulset']['elasticsearch-data']['spec']['template']['spec']['containers'][0]['env'] + assert {'name': 'discovery.zen.ping.unicast.hosts', + 'value': 'elasticsearch-master-headless'} in env + + +def test_set_master_service_to_default_nodegroup_name_with_custom_cluster_name(): + config = ''' +esMajorVersion: 6 +clusterName: "custom" +nodeGroup: "data" +''' + r = helm_template(config) + env = r['statefulset']['custom-data']['spec']['template']['spec']['containers'][0]['env'] + assert {'name': 'discovery.zen.ping.unicast.hosts', + 'value': 'custom-master-headless'} in env + + +def test_set_initial_master_nodes_when_using_v_7(): + config = ''' +esMajorVersion: 7 +roles: + master: "true" +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + assert { + 'name': 'cluster.initial_master_nodes', + 'value': 'elasticsearch-master-0,' + + 'elasticsearch-master-1,' + + 'elasticsearch-master-2,' + } in env + + for e in env: + assert e['name'] != 'discovery.zen.minimum_master_nodes' + + +def test_dont_set_initial_master_nodes_if_not_master_when_using_es_version_7(): + config = ''' +esMajorVersion: 7 +roles: + master: "false" +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + for e in env: + assert e['name'] != 'cluster.initial_master_nodes' + + +def test_set_discovery_seed_host_when_using_v_7(): + config = ''' +esMajorVersion: 7 +roles: + master: "true" +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + assert { + 'name': 'discovery.seed_hosts', + 'value': 'elasticsearch-master-headless' + } in env + + for e in env: + assert e['name'] != 'discovery.zen.ping.unicast.hosts' + + +def test_enabling_machine_learning_role(): + config = ''' +roles: + ml: "true" +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + + assert {'name': 'node.ml', + 'value': 'true'} in env + + +def test_adding_extra_env_vars(): + config = ''' +extraEnvs: + - name: hello + value: world +''' + r = helm_template(config) + env = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['env'] + assert {'name': 'hello', 'value': 'world'} in env + + +def test_adding_a_extra_volume_with_volume_mount(): + config = ''' +extraVolumes: | + - name: extras + emptyDir: {} +extraVolumeMounts: | + - name: extras + mountPath: /usr/share/extras + readOnly: true +''' + r = helm_template(config) + extraVolume = r['statefulset'][uname]['spec']['template']['spec']['volumes'] + assert {'name': 'extras', 'emptyDir': {}} in extraVolume + extraVolumeMounts = r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['volumeMounts'] + assert {'name': 'extras', 'mountPath': '/usr/share/extras', 'readOnly': True} in extraVolumeMounts + + +def test_adding_a_extra_init_container(): + config = ''' +extraInitContainers: | + - name: do-something + image: busybox + command: ['do', 'something'] +''' + r = helm_template(config) + extraInitContainer = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] + assert {'name': 'do-something', 'image': 'busybox', 'command': ['do', 'something'], } in extraInitContainer + +def test_sysctl_init_container_disabled(): + config = ''' +sysctlInitContainer: + enabled: false +''' + r = helm_template(config) + initContainers = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] + assert initContainers is None + +def test_sysctl_init_container_enabled(): + config = ''' +sysctlInitContainer: + enabled: true +''' + r = helm_template(config) + initContainers = r['statefulset'][uname]['spec']['template']['spec']['initContainers'] + assert initContainers[0]['name'] == 'configure-sysctl' + +def test_adding_storageclass_annotation_to_volumeclaimtemplate(): + config = ''' +persistence: + annotations: + volume.beta.kubernetes.io/storage-class: id +''' + r = helm_template(config) + annotations = r['statefulset'][uname]['spec']['volumeClaimTemplates'][0]['metadata']['annotations'] + assert annotations['volume.beta.kubernetes.io/storage-class'] == 'id' + +def test_adding_multiple_persistence_annotations(): + config = ''' + persistence: + annotations: + hello: world + world: hello + ''' + r = helm_template(config) + annotations = r['statefulset'][uname]['spec']['volumeClaimTemplates'][0]['metadata']['annotations'] + + assert annotations['hello'] == 'world' + assert annotations['world'] == 'hello' + + +def test_adding_a_secret_mount(): + config = ''' +secretMounts: + - name: elastic-certificates + secretName: elastic-certs + path: /usr/share/elasticsearch/config/certs +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/certs', + 'name': 'elastic-certificates' + } + assert s['volumes'] == [{ + 'name': 'elastic-certificates', + 'secret': { + 'secretName': 'elastic-certs' + } + }] + + +def test_adding_a_secret_mount_with_subpath(): + config = ''' +secretMounts: + - name: elastic-certificates + secretName: elastic-certs + path: /usr/share/elasticsearch/config/certs + subPath: cert.crt +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/certs', + 'subPath': 'cert.crt', + 'name': 'elastic-certificates' + } + + +def test_adding_image_pull_secrets(): + config = ''' +imagePullSecrets: + - name: test-registry +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['imagePullSecrets'][0]['name'] == 'test-registry' + + +def test_adding_tolerations(): + config = ''' +tolerations: +- key: "key1" + operator: "Equal" + value: "value1" + effect: "NoExecute" + tolerationSeconds: 3600 +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['tolerations'][0]['key'] == 'key1' + +def test_adding_pod_annotations(): + config = ''' +podAnnotations: + iam.amazonaws.com/role: es-role +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['metadata']['annotations']['iam.amazonaws.com/role'] == 'es-role' + + +def test_adding_a_node_selector(): + config = ''' +nodeSelector: + disktype: ssd +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['nodeSelector']['disktype'] == 'ssd' + +def test_adding_resources_to_initcontainer(): + config = ''' +initResources: + limits: + cpu: "25m" + memory: "128Mi" + requests: + cpu: "25m" + memory: "128Mi" +''' + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][0] + + assert i['resources'] == { + 'requests': { + 'cpu': '25m', + 'memory': '128Mi' + }, + 'limits': { + 'cpu': '25m', + 'memory': '128Mi' + } + } + +def test_adding_resources_to_sidecar_container(): + config = ''' +masterTerminationFix: true +sidecarResources: + limits: + cpu: "100m" + memory: "128Mi" + requests: + cpu: "100m" + memory: "128Mi" +''' + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['containers'][1] + + assert i['resources'] == { + 'requests': { + 'cpu': '100m', + 'memory': '128Mi' + }, + 'limits': { + 'cpu': '100m', + 'memory': '128Mi' + } + } + +def test_adding_a_node_affinity(): + config = ''' +nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: mylabel + operator: In + values: + - myvalue +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['affinity']['nodeAffinity'] == { + 'preferredDuringSchedulingIgnoredDuringExecution': [{ + 'weight': 100, + 'preference': { + 'matchExpressions': [{ + 'key': 'mylabel', + 'operator': 'In', + 'values': [ + 'myvalue' + ] + }] + } + }] + } + + +def test_adding_an_ingress_rule(): + config = ''' +ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + path: / + hosts: + - elasticsearch.elastic.co + tls: + - secretName: elastic-co-wildcard + hosts: + - elasticsearch.elastic.co +''' + + r = helm_template(config) + assert uname in r['ingress'] + i = r['ingress'][uname]['spec'] + assert i['tls'][0]['hosts'][0] == 'elasticsearch.elastic.co' + assert i['tls'][0]['secretName'] == 'elastic-co-wildcard' + + assert i['rules'][0]['host'] == 'elasticsearch.elastic.co' + assert i['rules'][0]['http']['paths'][0]['path'] == '/' + assert i['rules'][0]['http']['paths'][0]['backend']['serviceName'] == uname + assert i['rules'][0]['http']['paths'][0]['backend']['servicePort'] == 9200 + + +def test_changing_the_protocol(): + config = ''' +protocol: https +''' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert 'https://127.0.0.1:9200' in c['readinessProbe']['exec']['command'][-1] + +def test_changing_the_cluster_health_status(): + config = ''' +clusterHealthCheckParams: 'wait_for_no_initializing_shards=true&timeout=60s' +''' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert '/_cluster/health?wait_for_no_initializing_shards=true&timeout=60s' in c['readinessProbe']['exec']['command'][-1] + +def test_adding_in_es_config(): + config = ''' +esConfig: + elasticsearch.yml: | + key: + nestedkey: value + dot.notation: test + + log4j2.properties: | + appender.rolling.name = rolling +''' + r = helm_template(config) + c = r['configmap'][uname + '-config']['data'] + + assert 'elasticsearch.yml' in c + assert 'log4j2.properties' in c + + assert 'nestedkey: value' in c['elasticsearch.yml'] + assert 'dot.notation: test' in c['elasticsearch.yml'] + + assert 'appender.rolling.name = rolling' in c['log4j2.properties'] + + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert {'configMap': {'name': 'elasticsearch-master-config'}, 'name': 'esconfig'} in s['volumes'] + assert {'mountPath': '/usr/share/elasticsearch/config/elasticsearch.yml', 'name': 'esconfig', 'subPath': 'elasticsearch.yml'} in s['containers'][0]['volumeMounts'] + assert {'mountPath': '/usr/share/elasticsearch/config/log4j2.properties', 'name': 'esconfig', 'subPath': 'log4j2.properties'} in s['containers'][0]['volumeMounts'] + + assert 'configchecksum' in r['statefulset'][uname]['spec']['template']['metadata']['annotations'] + +def test_dont_add_data_volume_when_persistance_is_disabled(): + config = ''' +persistence: + enabled: false +''' + r = helm_template(config) + assert 'volumeClaimTemplates' not in r['statefulset'][uname]['spec'] + assert r['statefulset'][uname]['spec']['template']['spec']['containers'][0]['volumeMounts'] == None + + +def test_priority_class_name(): + config = ''' +priorityClassName: "" +''' + r = helm_template(config) + spec = r['statefulset'][uname]['spec']['template']['spec'] + assert 'priorityClassName' not in spec + + config = ''' +priorityClassName: "highest" +''' + r = helm_template(config) + priority_class_name = r['statefulset'][uname]['spec']['template']['spec']['priorityClassName'] + assert priority_class_name == "highest" + + +def test_scheduler_name(): + r = helm_template('') + spec = r['statefulset'][uname]['spec']['template']['spec'] + assert 'schedulerName' not in spec + + config = ''' +schedulerName: "stork" +''' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['schedulerName'] == "stork" + + +def test_adding_a_nodePort(): + config = '' + + r = helm_template(config) + + assert 'nodePort' not in r['service'][uname]['spec']['ports'][0] + + config = ''' + service: + nodePort: 30001 + ''' + + r = helm_template(config) + + assert r['service'][uname]['spec']['ports'][0]['nodePort'] == 30001 + +def test_master_termination_fixed_enabled(): + config = '' + + r = helm_template(config) + + assert len(r['statefulset'][uname]['spec']['template']['spec']['containers']) == 1 + + config = ''' + masterTerminationFix: true + ''' + + r = helm_template(config) + + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][1] + assert c['name'] == 'elasticsearch-master-graceful-termination-handler' + +def test_lifecycle_hooks(): + config = '' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert 'lifecycle' not in c + + config = ''' + lifecycle: + preStop: + exec: + command: ["/bin/bash","/preStop"] + ''' + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + + assert c['lifecycle']['preStop']['exec']['command'] == ["/bin/bash","/preStop"] + +def test_esMajorVersion_detect_default_version(): + config = '' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_default_to_7_if_not_elastic_image(): + config = ''' + image: notElastic + imageTag: 1.0.0 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_default_to_7_if_no_version_is_found(): + config = ''' + imageTag: not_a_number + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_set_to_6_based_on_image_tag(): + config = ''' + imageTag: 6.8.1 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' + +def test_esMajorVersion_always_wins(): + config = ''' + esMajorVersion: 7 + imageTag: 6.0.0 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '7' + +def test_esMajorVersion_parse_image_tag_for_oss_image(): + config = ''' + image: docker.elastic.co/elasticsearch/elasticsearch-oss + imageTag: 6.3.2 + ''' + + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['annotations']['esMajorVersion'] == '6' + +def test_set_pod_security_context(): + config = '' + r = helm_template(config) + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1000 + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['runAsUser'] == 1000 + + config = ''' + podSecurityContext: + fsGroup: 1001 + other: test + ''' + + r = helm_template(config) + + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1001 + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['other'] == 'test' + +def test_fsGroup_backwards_compatability(): + config = ''' + fsGroup: 1001 + ''' + + r = helm_template(config) + + assert r['statefulset'][uname]['spec']['template']['spec']['securityContext']['fsGroup'] == 1001 + +def test_set_container_security_context(): + config = '' + + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert c['securityContext']['capabilities']['drop'] == ['ALL'] + assert c['securityContext']['runAsNonRoot'] == True + assert c['securityContext']['runAsUser'] == 1000 + + config = ''' + securityContext: + runAsUser: 1001 + other: test + ''' + + r = helm_template(config) + c = r['statefulset'][uname]['spec']['template']['spec']['containers'][0] + assert c['securityContext']['capabilities']['drop'] == ['ALL'] + assert c['securityContext']['runAsNonRoot'] == True + assert c['securityContext']['runAsUser'] == 1001 + assert c['securityContext']['other'] == 'test' + +def test_adding_pod_labels(): + config = ''' +labels: + app.kubernetes.io/name: elasticsearch +''' + r = helm_template(config) + assert r['statefulset'][uname]['metadata']['labels']['app.kubernetes.io/name'] == 'elasticsearch' + +def test_keystore_enable(): + config = '' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert s['volumes'] == None + + config = ''' +keystore: + - secretName: test + ''' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert {'name': 'keystore', 'emptyDir': {}} in s['volumes'] + +def test_keystore_init_container(): + config = '' + + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1] + + assert i['name'] != 'keystore' + + config = ''' +keystore: + - secretName: test + ''' + + r = helm_template(config) + i = r['statefulset'][uname]['spec']['template']['spec']['initContainers'][-1] + + assert i['name'] == 'keystore' + +def test_keystore_mount(): + config = ''' +keystore: + - secretName: test +''' + + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['containers'][0]['volumeMounts'][-1] == { + 'mountPath': '/usr/share/elasticsearch/config/elasticsearch.keystore', + 'subPath': 'elasticsearch.keystore', + 'name': 'keystore' + } + +def test_keystore_init_volume_mounts(): + config = ''' +keystore: + - secretName: test + - secretName: test-with-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + assert s['initContainers'][-1]['volumeMounts'] == [ + { + 'mountPath': '/tmp/keystore', + 'name': 'keystore' + }, + { + 'mountPath': '/tmp/keystoreSecrets/test', + 'name': 'keystore-test' + }, + { + 'mountPath': '/tmp/keystoreSecrets/test-with-custom-path', + 'name': 'keystore-test-with-custom-path' + } + ] + +def test_keystore_volumes(): + config = ''' +keystore: + - secretName: test + - secretName: test-with-custom-path + items: + - key: slack_url + path: xpack.notification.slack.account.otheraccount.secure_url +''' + r = helm_template(config) + s = r['statefulset'][uname]['spec']['template']['spec'] + + assert { + 'name': 'keystore-test', + 'secret': { + 'secretName': 'test' + } + } in s['volumes'] + + assert { + 'name': 'keystore-test-with-custom-path', + 'secret': { + 'secretName': 'test-with-custom-path', + 'items': [{ + 'key': 'slack_url', + 'path': 'xpack.notification.slack.account.otheraccount.secure_url' + }] + } + } in s['volumes'] +def test_pod_security_policy(): + ## Make sure the default config is not creating any resources + config = '' + resources = ('role', 'rolebinding', 'serviceaccount', 'podsecuritypolicy') + r = helm_template(config) + for resource in resources: + assert resource not in r + assert 'serviceAccountName' not in r['statefulset'][uname]['spec']['template']['spec'] + + ## Make sure all the resources are created with default values + config = ''' +rbac: + create: true + serviceAccountName: "" + +podSecurityPolicy: + create: true + name: "" +''' + r = helm_template(config) + for resource in resources: + assert resource in r + assert r['role'][uname]['rules'][0] == {"apiGroups": ["extensions"], "verbs": ["use"], "resources": ["podsecuritypolicies"], "resourceNames": [uname]} + assert r['rolebinding'][uname]['subjects'] == [{"kind": "ServiceAccount", "namespace": "default", "name": uname}] + assert r['rolebinding'][uname]['roleRef'] == {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": uname} + assert r['statefulset'][uname]['spec']['template']['spec']['serviceAccountName'] == uname + psp_spec = r['podsecuritypolicy'][uname]['spec'] + assert psp_spec['privileged'] is True + + +def test_external_pod_security_policy(): + ## Make sure we can use an externally defined pod security policy + config = ''' +rbac: + create: true + serviceAccountName: "" + +podSecurityPolicy: + create: false + name: "customPodSecurityPolicy" +''' + resources = ('role', 'rolebinding') + r = helm_template(config) + for resource in resources: + assert resource in r + + assert r['role'][uname]['rules'][0] == {"apiGroups": ["extensions"], "verbs": ["use"], "resources": ["podsecuritypolicies"], "resourceNames": ["customPodSecurityPolicy"]} + + +def test_external_service_account(): + ## Make sure we can use an externally defined service account + config = ''' +rbac: + create: false + serviceAccountName: "customServiceAccountName" + +podSecurityPolicy: + create: false + name: "" +''' + resources = ('role', 'rolebinding', 'serviceaccount') + r = helm_template(config) + + assert r['statefulset'][uname]['spec']['template']['spec']['serviceAccountName'] == "customServiceAccountName" + # When referencing an external service account we do not want any resources to be created. + for resource in resources: + assert resource not in r diff --git a/kubernetes/helm_charts/logging/elasticsearch/values.j2 b/kubernetes/helm_charts/logging/elasticsearch/values.j2 new file mode 100755 index 000000000..29e5547ab --- /dev/null +++ b/kubernetes/helm_charts/logging/elasticsearch/values.j2 @@ -0,0 +1,241 @@ +--- +namespace: "logging" +clusterName: "elasticsearch" +nodeGroup: "master" + +# The service that non master groups will try to connect to when joining the cluster +# This should be set to clusterName + "-" + nodeGroup for your master group +masterService: "" + +# Elasticsearch roles that will be applied to this nodeGroup +# These will be set as environment variables. E.g. node.master=true +roles: + master: "true" + ingest: "true" + data: "true" + +replicas: 3 +minimumMasterNodes: 2 + +esMajorVersion: "" + +# Allows you to add any config files in /usr/share/elasticsearch/config/ +# such as elasticsearch.yml and log4j2.properties +esConfig: {} +# elasticsearch.yml: | +# key: +# nestedkey: value +# log4j2.properties: | +# key = value + +# Extra environment variables to append to this nodeGroup +# This will be appended to the current 'env:' key. You can use any of the kubernetes env +# syntax here +extraEnvs: [] +# - name: MY_ENVIRONMENT_VAR +# value: the_value_goes_here + +# A list of secrets and their paths to mount inside the pod +# This is useful for mounting certificates for security and for mounting +# the X-Pack license +secretMounts: [] +# - name: elastic-certificates +# secretName: elastic-certificates +# path: /usr/share/elasticsearch/config/certs + +image: "docker.io/keshavprasad/elasticsearch" +imageTag: "7.3.2" +imagePullPolicy: "IfNotPresent" + +podAnnotations: {} + # iam.amazonaws.com/role: es-cluster + +# additionals labels +labels: {} + +esJavaOpts: "-Xms3g -Xmx3g" + +resources: + requests: + cpu: "100m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "4Gi" + +initResources: {} + # limits: + # cpu: "25m" + # # memory: "128Mi" + # requests: + # cpu: "25m" + # memory: "128Mi" + +sidecarResources: {} + # limits: + # cpu: "25m" + # # memory: "128Mi" + # requests: + # cpu: "25m" + # memory: "128Mi" + +networkHost: "0.0.0.0" + +volumeClaimTemplate: + accessModes: [ "ReadWriteOnce" ] + storageClassName: "{{storage_class_name}}" + resources: + requests: + storage: 15Gi + +rbac: + create: true + serviceAccountName: "" + +podSecurityPolicy: + create: false + name: "" + spec: + privileged: true + fsGroup: + rule: RunAsAny + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - secret + - configMap + - persistentVolumeClaim + +persistence: + enabled: true + annotations: {} + +extraVolumes: "" + # - name: extras + # emptyDir: {} + +extraVolumeMounts: "" + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + +extraInitContainers: "" + # - name: do-something + # image: busybox + # command: ['do', 'something'] + +# This is the PriorityClass settings as defined in +# https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/#priorityclass +priorityClassName: "" + +# By default this will make sure two pods don't end up on the same node +# Changing this to a region would allow you to spread pods across regions +antiAffinityTopologyKey: "kubernetes.io/hostname" + +# Hard means that by default pods will only be scheduled if there are enough nodes for them +# and that they will never end up on the same node. Setting this to soft will do this "best effort" +antiAffinity: "soft" + +# This is the node affinity settings as defined in +# https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#node-affinity-beta-feature +nodeAffinity: {} + +# The default is to deploy all pods serially. By setting this to parallel all pods are started at +# the same time when bootstrapping the cluster +podManagementPolicy: "Parallel" + +protocol: http +httpPort: 9200 +transportPort: 9300 + +service: + type: ClusterIP + nodePort: + annotations: {} + httpPortName: http + transportPortName: transport + +updateStrategy: RollingUpdate + +# This is the max unavailable setting for the pod disruption budget +# The default value of 1 will make sure that kubernetes won't allow more than 1 +# of your pods to be unavailable during maintenance +maxUnavailable: 1 + +podSecurityContext: + fsGroup: 1000 + runAsUser: 1000 + +# The following value is deprecated, +# please use the above podSecurityContext.fsGroup instead +fsGroup: "" + +securityContext: + capabilities: + drop: + - ALL + # readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + +# How long to wait for elasticsearch to stop gracefully +terminationGracePeriod: 120 + +sysctlVmMaxMapCount: 262144 + +readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 3 + timeoutSeconds: 5 + +# https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html#request-params wait_for_status +clusterHealthCheckParams: "wait_for_status=green&timeout=1s" + +## Use an alternate scheduler. +## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/ +## +schedulerName: "" + +imagePullSecrets: [] +nodeSelector: {} +tolerations: [] + +# Enabling this will publically expose your Elasticsearch instance. +# Only enable this if you have security enabled on your cluster +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + path: / + hosts: + - chart-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +nameOverride: "" +fullnameOverride: "" + +# https://github.com/elastic/helm-charts/issues/63 +masterTerminationFix: false + +lifecycle: {} + # preStop: + # exec: + # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] + # postStart: + # exec: + # command: ["/bin/sh", "-c", "echo Hello from the postStart handler > /usr/share/message"] + +sysctlInitContainer: + enabled: true + +keystore: [] diff --git a/kubernetes/helm_charts/logging/fluent-bit/.helmignore b/kubernetes/helm_charts/logging/fluent-bit/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml b/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml new file mode 100644 index 000000000..47ff4d0df --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: fluent-bit +description: Fluentbit logging helm + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +appVersion: 1.6.3 diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl b/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl new file mode 100644 index 000000000..189773ee5 --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "fluent-bit.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fluent-bit.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fluent-bit.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "fluent-bit.labels" -}} +helm.sh/chart: {{ include "fluent-bit.chart" . }} +{{ include "fluent-bit.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "fluent-bit.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fluent-bit.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "fluent-bit.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "fluent-bit.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml b/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml new file mode 100644 index 000000000..8c3579e20 --- /dev/null +++ b/kubernetes/helm_charts/logging/fluent-bit/templates/configMap.yaml @@ -0,0 +1,166 @@ +--- +apiVersion: v1 +data: + filter-kubernetes.conf: | + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Kube_Tag_Prefix kube.var.log.containers. + # Merge application log to kubernetes structure + Merge_Log On + # Disabling merge key as all the fileds will be in the root + # Else all merged content will be under + # log_processed: + # { + # } + # Merge_Log_Key log_processed + # + # Adding custom parsor not to do any json parse + Merge_Parser catchall + # Don't need merged logs + # More over, we're using custom parser, which catch all and tag it as log + Keep_Log Off + # Annotate pod to select the parser + # annotaions: + # fluentbit.io/parsers: nginx2 + K8S-Logging.Parser On + # Disable fluent-bit from parsing logs + # annotaions: + # fluentbit.io/exclude: true + K8S-Logging.Exclude On + [FILTER] + # Discard all health debug and info logs + Name grep + Match kube.* + exclude message /^.*(?:debug|info|GET (\/service)?\/health).*$ + + fluent-bit.conf: | + [SERVICE] + Flush 1 + Log_Level WARNING + Daemon off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + storage.path /mnt/log/flb-storage/ + storage.sync normal + storage.checksum off + storage.backlog.mem_limit 120M + # How many chunks from file system, which is in que to flush to es, has to be in memory(warm) + storage.max_chunks_up 50 + + @INCLUDE input-kubernetes.conf + @INCLUDE filter-kubernetes.conf + @INCLUDE output-elasticsearch.conf + input-kubernetes.conf: | + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser docker + DB /mnt/log/flb_kube.db + Mem_Buf_Limit 120MB + # Set the initial buffer size to read files data. + # This value is used too to increase buffer size. + Buffer_Chunk_Size 256k + # Set the limit of the buffer size per monitored file. + # When a buffer needs to be increased (e.g: very long lines), + # this value is used to restrict how much the memory buffer can grow. + # If reading a file exceed this limit, the file is removed from the monitored file list + Buffer_Max_Size 512k + Skip_Long_Lines On + Refresh_Interval 1 + storage.type filesystem + # Ignore older logs of 30m + Ignore_Older 30m + output-elasticsearch.conf: | + [OUTPUT] + Name es + Match * + Host ${FLUENT_ELASTICSEARCH_HOST} + Port ${FLUENT_ELASTICSEARCH_PORT} + Logstash_Format On + Replace_Dots On + Retry_Limit 10 + Type _doc + # Networking Setup + net.connect_timeout 10 + # net.source_address 127.0.0.1 + net.keepalive on + net.keepalive_idle_timeout 100 + net.keepalive_max_recycle 200 + # + # Limit the maximum number of Chunks in the filesystem for the current output logical destination. + # After this LRU will kick in; that means LeastRecentlyUsed will get deleted + # 1 chunk =~ 2MB + # This config has some issues; see https://github.com/fluent/fluent-bit/issues/2688 + # storage.total_limit_size 20G + parsers.conf: | + [PARSER] + Name apache + Format regex + Regex ^(?[^ ]*) [^ ]* (?[^ ]*) \[(?