diff --git a/CHANGELOG.md b/CHANGELOG.md index f50afed..54716a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [v4.0.0-SNAPSHOT] - +- porting to keycloak ## [v3.2.0-SNAPSHOT] diff --git a/pom.xml b/pom.xml index 6c73acf..b17f221 100644 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ org.gcube.distribution gcube-bom - 2.1.0-SNAPSHOT + 3.0.0-SNAPSHOT pom import @@ -56,29 +56,11 @@ gcube-jackson-core - org.gcube.common - keycloak-client - [1.0.0,2.0.0-SNAPSHOT) + common-security - - - org.gcube.common - authorization-utils - [2.0.0-SNAPSHOT,3.0.0-SNAPSHOT) - - - - org.gcube.common - authorization-client - - - - org.gcube.common - common-authorization - - + org.gcube.data.publishing document-store-lib @@ -130,10 +112,23 @@ javax.servlet javax.servlet-api - 3.0.1 + 4.0.1 provided + + + io.micrometer + micrometer-core + 1.9.0 + + + + io.micrometer + micrometer-registry-prometheus + 1.9.0 + + diff --git a/src/main/java/org/gcube/smartgears/Bootstrap.java b/src/main/java/org/gcube/smartgears/Bootstrap.java index 1428436..f5df24a 100644 --- a/src/main/java/org/gcube/smartgears/Bootstrap.java +++ b/src/main/java/org/gcube/smartgears/Bootstrap.java @@ -14,6 +14,14 @@ import org.gcube.smartgears.provider.ProviderFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; + /** * Bootstraps management of all deployed applications which require it. * @@ -86,6 +94,17 @@ public class Bootstrap implements ServletContainerInitializer { /* Get the ContainerContext. Look at DefaultProvider */ context = ProviderFactory.provider().containerContext(); + PrometheusMeterRegistry registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + registry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT); + new ClassLoaderMetrics().bindTo(registry); + new JvmMemoryMetrics().bindTo(registry); + //new JvmGcMetrics().bindTo(registry); + new ProcessorMetrics().bindTo(registry); + new JvmThreadMetrics().bindTo(registry); + + Metrics.addRegistry(registry); + + /* Validate the configuration retrieved by ContainerContext * using gcube facilities annotation based * ( i.e org.gcube.common.validator.annotations) diff --git a/src/main/java/org/gcube/smartgears/extensions/resource/MetricsResource.java b/src/main/java/org/gcube/smartgears/extensions/resource/MetricsResource.java new file mode 100644 index 0000000..d2aa7f3 --- /dev/null +++ b/src/main/java/org/gcube/smartgears/extensions/resource/MetricsResource.java @@ -0,0 +1,44 @@ +package org.gcube.smartgears.extensions.resource; + +import static org.gcube.smartgears.Constants.plain_text; +import static org.gcube.smartgears.extensions.HttpExtension.Method.GET; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.gcube.smartgears.extensions.ApiResource; +import org.gcube.smartgears.extensions.ApiSignature; + +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.binder.jvm.ClassLoaderMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmGcMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmMemoryMetrics; +import io.micrometer.core.instrument.binder.jvm.JvmThreadMetrics; +import io.micrometer.core.instrument.binder.system.ProcessorMetrics; +import io.micrometer.prometheus.PrometheusConfig; +import io.micrometer.prometheus.PrometheusMeterRegistry; + +public class MetricsResource extends ApiResource { + + private static final long serialVersionUID = 1L; + + public static final String mapping = "/metrics"; + + private static final ApiSignature signature = handles(mapping).with(method(GET).produces(plain_text)); + + + MetricsResource() { + super(signature); + } + + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + PrometheusMeterRegistry registry = (PrometheusMeterRegistry) Metrics.globalRegistry.getRegistries().stream().findFirst().get(); + registry.scrape(resp.getWriter()); + } + +} \ No newline at end of file diff --git a/src/main/java/org/gcube/smartgears/extensions/resource/RemoteResource.java b/src/main/java/org/gcube/smartgears/extensions/resource/RemoteResource.java index 1acbf96..475c1d8 100644 --- a/src/main/java/org/gcube/smartgears/extensions/resource/RemoteResource.java +++ b/src/main/java/org/gcube/smartgears/extensions/resource/RemoteResource.java @@ -27,7 +27,7 @@ public class RemoteResource extends HttpController { public RemoteResource() { super(remote_management, default_mapping); addResources(new FrontPageResource(), new ConfigurationResource(), new ProfileResource(), - new LifecycleResource()); + new LifecycleResource(), new MetricsResource()); } @Override diff --git a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestAccounting.java b/src/main/java/org/gcube/smartgears/handlers/application/request/RequestAccounting.java index 3f7bab3..74f6f03 100644 --- a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestAccounting.java +++ b/src/main/java/org/gcube/smartgears/handlers/application/request/RequestAccounting.java @@ -1,14 +1,14 @@ package org.gcube.smartgears.handlers.application.request; +import java.util.concurrent.TimeUnit; + import javax.xml.bind.annotation.XmlRootElement; import org.gcube.accounting.datamodel.UsageRecord.OperationResult; import org.gcube.accounting.datamodel.usagerecords.ServiceUsageRecord; import org.gcube.accounting.persistence.AccountingPersistence; import org.gcube.accounting.persistence.AccountingPersistenceFactory; -import org.gcube.common.authorization.library.provider.AuthorizationProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider; -import org.gcube.common.authorization.utils.manager.SecretManagerProvider; import org.gcube.common.scope.api.ScopeProvider; import org.gcube.smartgears.Constants; import org.gcube.smartgears.configuration.Mode; @@ -20,23 +20,28 @@ import org.gcube.smartgears.utils.InnerMethodName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.Timer.Sample; + @XmlRootElement(name = Constants.request_accounting) public class RequestAccounting extends RequestHandler { private static Logger log = LoggerFactory.getLogger(RequestAccounting.class); private static ThreadLocal startCallThreadLocal = new ThreadLocal(); - - + + + @Override public String getName() { return Constants.request_accounting; } - + @Override public void handleRequest(RequestEvent e) { ApplicationContext context = e.context(); - + String calledMethod = e.request().getHeader(Constants.called_method_header); if (calledMethod==null){ calledMethod = e.request().getRequestURI().substring(e.request().getContextPath().length()); @@ -45,45 +50,58 @@ public class RequestAccounting extends RequestHandler { calledMethod= e.request().getMethod()+" "+calledMethod; } InnerMethodName.instance.set(calledMethod); - String caller = SecretManagerProvider.instance.get().getUser().getUsername(); + String caller = "Unknown"; startCallThreadLocal.set(System.currentTimeMillis()); + log.info("REQUEST START ON {}:{}({}) CALLED FROM {}@{} IN SCOPE {} ", context.configuration().name(),context.configuration().serviceClass(), InnerMethodName.instance.get(), caller, e.request().getRemoteHost(), ScopeProvider.instance.get()); - + } @Override public void handleResponse(ResponseEvent e) { ApplicationContext context = e.context(); - - boolean resetScope = false; - if (ScopeProvider.instance.get()==null && SecurityTokenProvider.instance.get()==null){ - String infrastructure = e.context().container().configuration().infrastructure(); - ScopeProvider.instance.set("/"+infrastructure); - resetScope = true; + + try { + boolean resetScope = false; + if (ScopeProvider.instance.get()==null && SecurityTokenProvider.instance.get()==null){ + String infrastructure = e.context().container().configuration().infrastructure(); + ScopeProvider.instance.set("/"+infrastructure); + resetScope = true; + } + + String caller = "Unknown"; + String callerQualifier = "UNKNOWN"; + //retieves caller Ip when there is a proxy + String callerIp = e.request().getHeader("x-forwarded-for"); + if(callerIp==null) + callerIp=e.request().getRemoteHost(); + + boolean success = e.response().getStatus()<400; + + if (context.container().configuration().mode()!=Mode.offline) + generateAccounting(caller,callerQualifier,callerIp==null?"UNKNOWN":callerIp , success, context); + + long durationInMillis = System.currentTimeMillis()-startCallThreadLocal.get(); + + Metrics.globalRegistry.timer("http.requests", "response",Integer.toString(e.response().getStatus()) + , "context", ScopeProvider.instance.get(), "result", success?"SUCCEDED":"FAILED", "caller-ip", callerIp, + "caller-username", caller, "service-class", context.configuration().serviceClass(), "service-name", context.configuration().name(), + "method", InnerMethodName.instance.get()).record(durationInMillis, TimeUnit.MILLISECONDS); + + log.info("REQUEST SERVED ON {}:{}({}) CALLED FROM {}@{} IN SCOPE {} {}(CODE {}) IN {} millis", + context.configuration().name(),context.configuration().serviceClass(), InnerMethodName.instance.get(), + caller, callerIp, ScopeProvider.instance.get(), success?"SUCCEDED":"FAILED", e.response().getStatus(),durationInMillis); + startCallThreadLocal.remove(); + InnerMethodName.instance.reset(); + if (resetScope) + ScopeProvider.instance.reset(); + }catch (Exception e1) { + log.error("error on accounting",e); + throw e1; } - - String caller = SecretManagerProvider.instance.get().getUser().getUsername(); - String callerQualifier = "UNKNOWN"; - //retieves caller Ip when there is a proxy - String callerIp = e.request().getHeader("x-forwarded-for"); - if(callerIp==null) - callerIp=e.request().getRemoteHost(); - - boolean success = e.response().getStatus()<400; - - if (context.container().configuration().mode()!=Mode.offline) - generateAccounting(caller,callerQualifier,callerIp==null?"UNKNOWN":callerIp , success, context); - - log.info("REQUEST SERVED ON {}:{}({}) CALLED FROM {}@{} IN SCOPE {} {}(CODE {}) IN {} millis", - context.configuration().name(),context.configuration().serviceClass(), InnerMethodName.instance.get(), - caller, callerIp, ScopeProvider.instance.get(), success?"SUCCEDED":"FAILED", e.response().getStatus(), System.currentTimeMillis()-startCallThreadLocal.get()); - startCallThreadLocal.remove(); - InnerMethodName.instance.reset(); - if (resetScope) - ScopeProvider.instance.reset(); - + } void generateAccounting(String caller, String callerQualifier, String remoteHost, boolean success, ApplicationContext context){ @@ -91,20 +109,20 @@ public class RequestAccounting extends RequestHandler { AccountingPersistence persistence = AccountingPersistenceFactory.getPersistence(); ServiceUsageRecord serviceUsageRecord = new ServiceUsageRecord(); try{ - + serviceUsageRecord.setConsumerId(caller); serviceUsageRecord.setCallerQualifier(callerQualifier); serviceUsageRecord.setScope(ScopeProvider.instance.get()); serviceUsageRecord.setServiceClass(context.configuration().serviceClass()); serviceUsageRecord.setServiceName(context.configuration().name()); - + serviceUsageRecord.setHost(context.container().configuration().hostname()+":"+context.container().configuration().port()); serviceUsageRecord.setCalledMethod(InnerMethodName.instance.get()); serviceUsageRecord.setCallerHost(remoteHost); serviceUsageRecord.setOperationResult(success?OperationResult.SUCCESS:OperationResult.FAILED); serviceUsageRecord.setDuration(System.currentTimeMillis()-startCallThreadLocal.get()); persistence.account(serviceUsageRecord); - + }catch(Exception ex){ log.warn("invalid record passed to accounting ",ex); } @@ -115,6 +133,6 @@ public class RequestAccounting extends RequestHandler { return getName(); } - - + + } diff --git a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestContextRetriever.java b/src/main/java/org/gcube/smartgears/handlers/application/request/RequestContextRetriever.java deleted file mode 100644 index 787baaf..0000000 --- a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestContextRetriever.java +++ /dev/null @@ -1,102 +0,0 @@ -package org.gcube.smartgears.handlers.application.request; - -import static org.gcube.smartgears.Constants.scope_header; -import static org.gcube.smartgears.Constants.token_header; -import static org.gcube.smartgears.handlers.application.request.RequestError.internal_server_error; - -import java.util.Base64; - -import javax.xml.bind.annotation.XmlRootElement; - -import org.gcube.common.authorization.utils.manager.SecretManager; -import org.gcube.common.authorization.utils.manager.SecretManagerProvider; -import org.gcube.common.authorization.utils.secret.GCubeSecret; -import org.gcube.common.authorization.utils.secret.JWTSecret; -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.smartgears.Constants; -import org.gcube.smartgears.handlers.application.RequestEvent; -import org.gcube.smartgears.handlers.application.RequestHandler; -import org.gcube.smartgears.handlers.application.ResponseEvent; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@XmlRootElement(name = Constants.request_context_retriever) -public class RequestContextRetriever extends RequestHandler { - - private static Logger log = LoggerFactory.getLogger(RequestContextRetriever.class); - - private static final String BEARER_AUTH_PREFIX ="Bearer"; - private static final String BASIC_AUTH_PREFIX ="Basic"; - - - @Override - public String getName() { - return Constants.request_context_retriever; - } - - @Override - public void handleRequest(RequestEvent call) { - String token = call.request().getParameter(token_header)==null? call.request().getHeader(token_header):call.request().getParameter(token_header); - String scope = call.request().getParameter(scope_header)==null? call.request().getHeader(scope_header):call.request().getParameter(scope_header); - - String authHeader = call.request().getHeader(Constants.authorization_header); - - log.trace("authorization header is {}",authHeader); - log.trace("token header is {}", token); - log.trace("scope header is {}", scope); - - String retrievedUser = null; - String accessToken = null; - if (authHeader!=null && !authHeader.isEmpty()) { - if (authHeader.startsWith(BEARER_AUTH_PREFIX)) - accessToken = authHeader.substring(BEARER_AUTH_PREFIX.length()).trim(); - else if (token==null && authHeader.startsWith(BASIC_AUTH_PREFIX)) { - String basicAuthToken = authHeader.substring(BASIC_AUTH_PREFIX.length()).trim(); - String decodedAuth = new String(Base64.getDecoder().decode(basicAuthToken.getBytes())); - String[] splitAuth = decodedAuth.split(":"); - token = splitAuth[1]; - retrievedUser = splitAuth[0]; - } - } - - SecretManager secretManager = new SecretManager(); - SecretManagerProvider.instance.set(secretManager); - - if (accessToken!=null) { - JWTSecret jwtSecret = new JWTSecret(accessToken); - secretManager.addSecret(jwtSecret); - } - - if (token!=null) { - GCubeSecret gCubeSecret = new GCubeSecret(token); - secretManager.addSecret(gCubeSecret); - try { - if (retrievedUser != null && !gCubeSecret.getClientInfo().getId().equals(retrievedUser)) { - internal_server_error.fire("user and token owner are not the same"); - } - }catch (Exception e) { - internal_server_error.fire(e.getMessage()); - } - } - - if(accessToken==null && token==null) { - if(scope!=null) { - ScopeProvider.instance.set(scope); - } - }else { - try { - secretManager.set(); - } catch (Exception e) { - internal_server_error.fire(e.getMessage()); - } - } - - } - - @Override - public void handleResponse(ResponseEvent e) { - log.debug("resetting all the Thread local for this call."); - SecretManagerProvider.instance.reset(); - } - -} \ No newline at end of file diff --git a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestValidator.java b/src/main/java/org/gcube/smartgears/handlers/application/request/RequestValidator.java index dff908f..c8a39b3 100644 --- a/src/main/java/org/gcube/smartgears/handlers/application/request/RequestValidator.java +++ b/src/main/java/org/gcube/smartgears/handlers/application/request/RequestValidator.java @@ -1,34 +1,37 @@ package org.gcube.smartgears.handlers.application.request; +import static org.gcube.smartgears.Constants.token_header; import static org.gcube.smartgears.handlers.application.request.RequestError.application_failed_error; import static org.gcube.smartgears.handlers.application.request.RequestError.application_unavailable_error; import static org.gcube.smartgears.handlers.application.request.RequestError.invalid_request_error; -import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlRootElement; -import org.gcube.common.authorization.utils.manager.SecretManager; -import org.gcube.common.authorization.utils.manager.SecretManagerProvider; +import javax.xml.crypto.dsig.keyinfo.RetrievalMethod; + import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.impl.ScopeBean; import org.gcube.common.scope.impl.ScopeBean.Type; +import org.gcube.common.security.providers.SecretManagerProvider; +import org.gcube.common.security.secrets.GCubeSecret; +import org.gcube.common.security.secrets.JWTSecret; +import org.gcube.common.security.secrets.Secret; import org.gcube.smartgears.Constants; import org.gcube.smartgears.configuration.Mode; import org.gcube.smartgears.configuration.container.ContainerConfiguration; import org.gcube.smartgears.context.application.ApplicationContext; import org.gcube.smartgears.handlers.application.RequestEvent; import org.gcube.smartgears.handlers.application.RequestHandler; +import org.gcube.smartgears.handlers.application.ResponseEvent; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @XmlRootElement(name = Constants.request_validation) public class RequestValidator extends RequestHandler { - @XmlAttribute(required=false, name="oauth") - @Deprecated - boolean oauthCompatibility = false; - private static Logger log = LoggerFactory.getLogger(RequestValidator.class); + private static final String BEARER_AUTH_PREFIX ="Bearer"; + private ApplicationContext appContext; @Override @@ -42,7 +45,9 @@ public class RequestValidator extends RequestHandler { log.trace("executing request validator ON REQUEST"); appContext = call.context(); - + + SecretManagerProvider.instance.set(getSecret(call)); + validateAgainstLifecycle(call); rejectUnauthorizedCalls(call); @@ -54,6 +59,13 @@ public class RequestValidator extends RequestHandler { } + @Override + public void handleResponse(ResponseEvent e) { + log.debug("resetting all the Thread local for this call."); + SecretManagerProvider.instance.reset(); + } + + private void validateAgainstLifecycle(RequestEvent call) { switch(appContext.lifecycle().state()) { @@ -68,7 +80,6 @@ public class RequestValidator extends RequestHandler { //nothing to do, but avoids warnings } - } private void validateScopeCall() { @@ -92,10 +103,10 @@ public class RequestValidator extends RequestHandler { } private void rejectUnauthorizedCalls(RequestEvent call){ - - SecretManager secretManager = SecretManagerProvider.instance.get(); - - if (secretManager.getCurrentSecretHolder().getSecrets().size()>0){ + + Secret secret = SecretManagerProvider.instance.get(); + + if (secret!= null){ log.warn("rejecting call to {}, authorization required",appContext.name()); RequestError.request_not_authorized_error.fire(appContext.name()+": authorization required"); } @@ -109,8 +120,26 @@ public class RequestValidator extends RequestHandler { private void validatePolicy(String scope, RequestEvent call){ //TODO: must be re-think } - + private Secret getSecret(RequestEvent call){ + String token = call.request().getParameter(token_header)==null? call.request().getHeader(token_header):call.request().getParameter(token_header); + String authHeader = call.request().getHeader(Constants.authorization_header); + + log.trace("authorization header is {}",authHeader); + log.trace("token header is {}", token); + + String accessToken = null; + if (authHeader!=null && !authHeader.isEmpty()) + if (authHeader.startsWith(BEARER_AUTH_PREFIX)) + accessToken = authHeader.substring(BEARER_AUTH_PREFIX.length()).trim(); + + Secret secret = null; + if (accessToken!=null) + secret = new JWTSecret(accessToken); + else if (token!=null) + secret = new GCubeSecret(token); + return secret; + } } diff --git a/src/main/java/org/gcube/smartgears/managers/ContainerManager.java b/src/main/java/org/gcube/smartgears/managers/ContainerManager.java index 96e777e..4323232 100644 --- a/src/main/java/org/gcube/smartgears/managers/ContainerManager.java +++ b/src/main/java/org/gcube/smartgears/managers/ContainerManager.java @@ -24,7 +24,6 @@ import org.gcube.smartgears.handlers.container.ContainerLifecycleEvent; import org.gcube.smartgears.handlers.container.ContainerPipeline; import org.gcube.smartgears.lifecycle.application.ApplicationLifecycle; import org.gcube.smartgears.lifecycle.container.ContainerState; -import org.gcube.smartgears.security.AuthorizationProvider; import org.gcube.smartgears.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/gcube/smartgears/provider/DefaultProvider.java b/src/main/java/org/gcube/smartgears/provider/DefaultProvider.java index 334336b..e51d90e 100644 --- a/src/main/java/org/gcube/smartgears/provider/DefaultProvider.java +++ b/src/main/java/org/gcube/smartgears/provider/DefaultProvider.java @@ -132,7 +132,7 @@ public class DefaultProvider implements Provider { try { - InputStream config = getClass().getResourceAsStream(container_handlers_file_path); + //TODO retrieve handler classes if (config == null) throw new IllegalStateException("invalid distribution: cannot find " + container_handlers_file_path); diff --git a/src/main/resources/META-INF/default-handlers.xml b/src/main/resources/META-INF/default-handlers.xml index 39ecdba..d6a1181 100644 --- a/src/main/resources/META-INF/default-handlers.xml +++ b/src/main/resources/META-INF/default-handlers.xml @@ -4,7 +4,6 @@ - diff --git a/src/main/resources/META-INF/profile.xml b/src/main/resources/META-INF/profile.xml new file mode 100644 index 0000000..3bc929b --- /dev/null +++ b/src/main/resources/META-INF/profile.xml @@ -0,0 +1,29 @@ + + + + + Service + + ${description} + ${serviceClass} + common-smartgears + 1.0.0 + + + ${description} + common-smartgears + 4.0.0-SNAPSHOT + + org.gcube.core + common-smartgears + 4.0.0-SNAPSHOT + + Library + + common-smartgears-4.0.0-SNAPSHOT.jar + + + + + +