Keycloak and external Infinispan and Console -> throws error while marshaling

2k Views Asked by At

I have configured setup based on example from https://github.com/thomasdarimont/keycloak-project-example/tree/main/deployments/local/clusterx/haproxy-external-ispn-database which in my case works on local kubernetes (minikube).

My keycloak (quarkus version 20.0.3) infinispan subsystem looks like below:

<?xml version="1.0" encoding="UTF-8"?>
<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="urn:infinispan:config:13.0 http://www.infinispan.org/schemas/infinispan-config-13.0.xsd"
        xmlns="urn:infinispan:config:13.0">

    <cache-container name="keycloak">
        <transport lock-timeout="60000"/>

        <local-cache-configuration name="local-cache-cfg">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
        </local-cache-configuration>

        <local-cache name="realms" configuration="local-cache-cfg">
            <memory max-count="10000"/>
        </local-cache>

        <local-cache name="users" configuration="local-cache-cfg">
            <memory max-count="10000"/>
        </local-cache>


        <local-cache name="keys" configuration="local-cache-cfg">
            <expiration max-idle="3600000"/>
            <memory max-count="1000"/>
        </local-cache>

        <local-cache name="authorization" configuration="local-cache-cfg">
            <memory max-count="10000"/>
        </local-cache>

        <distributed-cache name="authenticationSessions" owners="2">
            <expiration lifespan="-1"/>
        </distributed-cache>

        <distributed-cache name="sessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                          cache="sessions"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>

                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="clientSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                          cache="clientSessions"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="offlineSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                          cache="offlineSessions"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" owners="2">
            <expiration lifespan="-1"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                          cache="offlineClientSessions"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="loginFailures" owners="2">
            <expiration lifespan="-1"/>
            <remote-store xmlns="urn:infinispan:config:store:remote:13.0"
                          cache="loginFailures"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <distributed-cache name="actionTokens" owners="2">
            <encoding>
                <key media-type="application/x-java-object"/>
                <value media-type="application/x-java-object"/>
            </encoding>
            <expiration max-idle="-1" lifespan="-1" interval="300000"/>
            <memory max-count="-1"/>
            <remote-store cache="actionTokens" xmlns="urn:infinispan:config:store:remote:13.0"
                          fetch-state="false"
                          purge="false"
                          preload="false"
                          shared="true"
                          segmented="false"
                          connect-timeout="${env.KEYCLOAK_REMOTE_ISPN_CONN_TIMEOUT:2000}"
                          raw-values="true"
                          marshaller="org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory">
                <remote-server host="external-infinispan" port="${infinispan.bind.port:8766}"/>
                <security>
                    <authentication>
                        <digest username="${env.KEYCLOAK_REMOTE_ISPN_USERNAME:keycloak}"
                                password="${env.KEYCLOAK_REMOTE_ISPN_PASSWORD:password}"
                                realm="default"/>
                    </authentication>
                    <encryption>
                        <truststore filename="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PATH:/opt/keycloak/conf/ispn-truststore.jks}"
                                    password="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_PASSWORD:password}"
                                    type="${env.KEYCLOAK_REMOTE_ISPN_TRUSTSTORE_TYPE:JKS}"/>
                    </encryption>
                </security>
            </remote-store>
        </distributed-cache>

        <replicated-cache name="work">
            <expiration lifespan="-1"/>
        </replicated-cache>
    </cache-container>
</infinispan>

and my external infinispan (version 14.0.7-Final-2) configuration looks like below:

<infinispan
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xmlns="urn:infinispan:config:13.0"
        xsi:schemaLocation="urn:infinispan:config:13.0 https://infinispan.org/schemas/infinispan-config-13.0.xsd urn:infinispan:server:13.0 https://infinispan.org/schemas/infinispan-server-13.0.xsd">

    <!-- TODO configure JGROUPS tcp Stack with encryption -->

    <!-- see https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-config-13.0.html -->


    <cache-container name="default" statistics="true">

        <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
            <allow-list>
                <class>org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper</class>
                <regex>org.keycloak.models.sessions.infinispan.changes.*</regex>
            </allow-list>
        </serialization> 

        <!-- TODO configure custom jgroups stack: +auth +encryption -->
        <transport
                cluster="${infinispan.cluster.name:REMOTE}"
                stack="${infinispan.cluster.stack:kubernetes}"
                node-name="${infinispan.node.name:}"/>

        <replicated-cache-configuration name="replicated-cache-cfg"
                                        xmlns:jdbc="urn:infinispan:config:store:jdbc:13.0"
                                        mode="SYNC"
                                        statistics="true"
                                        segments="256"
                                        unreliable-return-values="false">
           <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>
            <transaction mode="NON_XA"
                         locking="OPTIMISTIC"/>
            <persistence passivation="false">
                <jdbc:string-keyed-jdbc-store fetch-state="false" shared="true" preload="false">
                    <jdbc:data-source jndi-url="jdbc/datasource"/>
                    <jdbc:string-keyed-table drop-on-exit="false" create-on-start="true" prefix="ispn">
                        <jdbc:id-column name="id" type="VARCHAR(255)"/>
                        <jdbc:data-column name="data" type="bytea"/>
                        <jdbc:timestamp-column name="ts" type="BIGINT"/>
                        <jdbc:segment-column name="seg" type="INT"/>
                    </jdbc:string-keyed-table>
                </jdbc:string-keyed-jdbc-store>
            </persistence>
        </replicated-cache-configuration>

        <distributed-cache-configuration name="distributed-cache-cfg"
                                         xmlns:jdbc="urn:infinispan:config:store:jdbc:13.0"
                                         mode="SYNC"
                                         owners="2"
                                         remote-timeout="60000"
                                         statistics="true"
                                         segments="256"
                                         unreliable-return-values="false">

            <encoding>
                <key media-type="application/x-jboss-marshalling"/>
                <value media-type="application/x-jboss-marshalling"/>
            </encoding>
            <locking isolation="REPEATABLE_READ"
                     striping="false"
                     acquire-timeout="10000"
                     concurrency-level="32"/>

            <transaction mode="NON_XA"
                         locking="OPTIMISTIC"/>

            <expiration lifespan="-1"
                        max-idle="-1"
                        interval="60000" />

            <memory max-count="-1" when-full="NONE" storage="HEAP"/>

            <partition-handling when-split="ALLOW_READ_WRITES" />

            <persistence passivation="false">
                <jdbc:string-keyed-jdbc-store fetch-state="false" shared="true" preload="false">
                    <jdbc:data-source jndi-url="jdbc/datasource"/>
                    <jdbc:string-keyed-table drop-on-exit="false" create-on-start="true" prefix="ispn">
                        <jdbc:id-column name="id" type="VARCHAR(255)"/>
                        <jdbc:data-column name="data" type="bytea"/>
                        <jdbc:timestamp-column name="ts" type="BIGINT"/>
                        <jdbc:segment-column name="seg" type="INT"/>
                    </jdbc:string-keyed-table>
                </jdbc:string-keyed-jdbc-store>
            </persistence>


            <state-transfer enabled="true"
                            timeout="240000"
                            chunk-size="240000"
                            await-initial-transfer="true"/>
        </distributed-cache-configuration>

<!--        <replicated-cache name="work" configuration="replicated-cache-cfg">-->
<!--        </replicated-cache>-->

        <distributed-cache name="sessions" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="authenticationSessions" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="offlineSessions" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="clientSessions" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="offlineClientSessions" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="loginFailures" configuration="distributed-cache-cfg">
        </distributed-cache>

        <distributed-cache name="actionTokens" configuration="distributed-cache-cfg">
            <memory max-count="-1">
            </memory>
            <expiration interval="300000" max-idle="-1"/>
        </distributed-cache>

<!--        <security>-->
<!--            <authorization>-->
<!--                <roles>-->
<!--                    <role name="supervisor" permissions="READ WRITE EXEC CREATE"/>-->
<!--                </roles>-->
<!--            </authorization>-->
<!--        </security>-->
    </cache-container>

    <!-- https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-server-13.0.html# -->
    <server xmlns="urn:infinispan:server:13.0">

        <interfaces>
            <interface name="public">
                <!-- we bind to the eth0 interface instead of a specific ip address to ease access -->
                <!--                <inet-address value="${infinispan.bind.address:127.0.0.1}"/>-->
                <match-interface value="eth0"/>
                <!-- or use any-address element -->
            </interface>
        </interfaces>

        <socket-bindings default-interface="public" port-offset="${infinispan.socket.binding.port-offset:0}">
            <socket-binding name="default" port="${infinispan.bind.port:11222}"/>
<!--            <socket-binding name="memcached" port="11221"/>-->
        </socket-bindings>

        <security>
            <security-realms>
                <security-realm name="default">
                    <!--  Uncomment to enable TLS on the realm  -->
                    <server-identities>
                        <ssl>
                            <keystore path="ispn-server.jks" relative-to="infinispan.server.config.path"
                                      password="password" alias="server" key-password="password"
                                      generate-self-signed-certificate-host="localhost"/>
                        </ssl>
                    </server-identities>
                    <properties-realm groups-attribute="Roles">
                        <user-properties path="users.properties" relative-to="infinispan.server.config.path"
                                         plain-text="true"/>
                        <group-properties path="groups.properties" relative-to="infinispan.server.config.path"/>
                    </properties-realm>
                </security-realm>
            </security-realms>
        </security>


        <data-sources>
            <data-source name="KeycloakDS" jndi-name="jdbc/datasource" statistics="true">
                <connection-factory driver="org.postgresql.Driver"
                                    username="postgres"
                                    password="password"
                                    url="jdbc:postgresql://db-address/postgres"
                                    new-connection-sql="SELECT 1" transaction-isolation="READ_COMMITTED">
                </connection-factory>
                <connection-pool initial-size="1" max-size="10"  min-size="3" background-validation="1000" idle-removal="1" blocking-timeout="1000" leak-detection="10000"/>
            </data-source>
        </data-sources>

        <!-- see https://docs.jboss.org/infinispan/13.0/configdocs/infinispan-server-13.0.html#endpoints -->
        <endpoints>
            <endpoint socket-binding="default" security-realm="default">
                <hotrod-connector name="hotrod" security-realm="default"/>
                <rest-connector>
                    <authentication mechanisms="BASIC"/>
                </rest-connector>
            </endpoint>
        </endpoints>
    </server>

</infinispan>

I can see that almost all functionalities seems to work pretty well, but when I navigate to infinispan console Global Statistics and then click cache which has entries then infinispan throws exception:


[org.infinispan.rest.stream.CacheChunkedStream] Error encountered while streaming cache chunk org.infinispan.commons.dataconversion.EncodingException: ISPN000495: JBossMarshallingTranscoder encountered error transcoding content
 at org.infinispan.jboss.marshalling.dataconversion.JBossMarshallingTranscoder.unmarshall(JBossMarshallingTranscoder.java:84)
 at org.infinispan.jboss.marshalling.dataconversion.JBossMarshallingTranscoder.doTranscode(JBossMarshallingTranscoder.java:59)
 at org.infinispan.commons.dataconversion.AbstractTranscoder.transcode(AbstractTranscoder.java:69)
 at org.infinispan.encoding.DataConversion.fromStorage(DataConversion.java:222)
 at org.infinispan.cache.impl.EncoderEntryMapper.apply(EncoderEntryMapper.java:63)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableMap$MapSubscriber.onNext(FlowableMap.java:64)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableRepeatUntil$RepeatSubscriber.onNext(FlowableRepeatUntil.java:67)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableDoOnLifecycle$SubscriptionLambdaSubscriber.onNext(FlowableDoOnLifecycle.java:80)
 at io.reactivex.rxjava3.processors.UnicastProcessor.drainRegular(UnicastProcessor.java:309)
 at io.reactivex.rxjava3.processors.UnicastProcessor.drain(UnicastProcessor.java:384)
 at io.reactivex.rxjava3.processors.UnicastProcessor.onNext(UnicastProcessor.java:444)
 at org.infinispan.reactive.publisher.impl.InnerPublisherSubscription$InnerPublisherSubscriptionBuilder$1.doOnValue(InnerPublisherSubscription.java:81)
 at org.infinispan.reactive.publisher.impl.InnerPublisherSubscription.lambda$accept$0(InnerPublisherSubscription.java:228)
 at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863)
 at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841)
 at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510)
 at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2147)
 at org.infinispan.reactive.publisher.impl.PublisherHandler$PublisherState.prepareResponse(PublisherHandler.java:438)
 at org.infinispan.reactive.publisher.impl.PublisherHandler$PublisherState.onComplete(PublisherHandler.java:301)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatMap$ConcatMapImmediate.drain(FlowableConcatMap.java:258)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatMap$BaseConcatMapSubscriber.onComplete(FlowableConcatMap.java:165)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableDoFinally$DoFinallySubscriber.onComplete(FlowableDoFinally.java:96)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatMap$ConcatMapImmediate.drain(FlowableConcatMap.java:258)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatMap$BaseConcatMapSubscriber.innerComplete(FlowableConcatMap.java:171)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatMap$ConcatMapInner.onComplete(FlowableConcatMap.java:587)
 at io.reactivex.rxjava3.internal.subscribers.SinglePostCompleteSubscriber.complete(SinglePostCompleteSubscriber.java:82)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatWithSingle$ConcatWithSubscriber.onSuccess(FlowableConcatWithSingle.java:81)
 at io.reactivex.rxjava3.internal.operators.single.SingleFromSupplier.subscribeActual(SingleFromSupplier.java:60)
 at io.reactivex.rxjava3.core.Single.subscribe(Single.java:4855)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatWithSingle$ConcatWithSubscriber.onComplete(FlowableConcatWithSingle.java:89)
 at io.reactivex.rxjava3.internal.subscribers.BasicFuseableSubscriber.onComplete(BasicFuseableSubscriber.java:120)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableConcatArray$ConcatArraySubscriber.onComplete(FlowableConcatArray.java:113)
 at io.reactivex.rxjava3.internal.subscribers.BasicFuseableSubscriber.onComplete(BasicFuseableSubscriber.java:120)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableUsing$UsingSubscriber.onComplete(FlowableUsing.java:144)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableObserveOn$BaseObserveOnSubscriber.checkTerminated(FlowableObserveOn.java:217)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableObserveOn$ObserveOnSubscriber.runAsync(FlowableObserveOn.java:396)
 at io.reactivex.rxjava3.internal.operators.flowable.FlowableObserveOn$BaseObserveOnSubscriber.run(FlowableObserveOn.java:178)
 at io.reactivex.rxjava3.internal.schedulers.ExecutorScheduler$ExecutorWorker$BooleanRunnable.run(ExecutorScheduler.java:324)
 at io.reactivex.rxjava3.internal.schedulers.ExecutorScheduler$ExecutorWorker.runEager(ExecutorScheduler.java:289)
 at io.reactivex.rxjava3.internal.schedulers.ExecutorScheduler$ExecutorWorker.run(ExecutorScheduler.java:250)
 at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)
 at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)
 at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)
 at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:403)
 at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)
 at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
 at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: java.lang.ClassNotFoundException: org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper
 at java.base/java.net.URLClassLoader.findClass(URLClassLoader.java:445)
 at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:587)
 at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:520)
 at java.base/java.lang.Class.forName0(Native Method)
 at java.base/java.lang.Class.forName(Class.java:467)
 at org.jboss.marshalling.AbstractClassResolver.loadClass(AbstractClassResolver.java:129)
 at org.jboss.marshalling.AbstractClassResolver.resolveClass(AbstractClassResolver.java:110)
 at org.infinispan.jboss.marshalling.commons.CheckedClassResolver.resolveClass(CheckedClassResolver.java:31)
.
.
.

Seems like this java.lang.ClassNotFoundException: org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper directly says that infinispan does not have such class.

Do you know how to fix that? Should I try with different version of Infinispan?

1

There are 1 best solutions below

0
lwalukie On

If anyone faces similiar issues below are things which I had to do to make it works.

According to pruivo hints I applied following changes:

  1. Added keycloak files to /opt/infinispan/server/lib folder of infinispan image.
FROM quay.io/infinispan/server:14.0

COPY /org.jboss.marshalling.jboss-marshalling-osgi-2.1.1.Final.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-admin-ui-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-authz-policy-common-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-common-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-config-api-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-core-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-crypto-default-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-crypto-fips1402-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-js-adapter-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-kerberos-federation-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-ldap-federation-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-infinispan-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-jpa-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-legacy-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-legacy-private-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-legacy-services-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-map-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-map-file-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-map-hot-rod-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-model-map-jpa-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-rest-admin-ui-ext-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-saml-core-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-saml-core-public-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-server-spi-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-server-spi-private-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-services-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-sssd-federation-21.0.1.jar /opt/infinispan/server/lib
COPY /org.keycloak.keycloak-themes-21.0.1.jar /opt/infinispan/server/lib

COPY /common.sh /opt/infinispan/bin

USER 0

RUN true \
  && microdnf clean all \
  && microdnf install shadow-utils \
  && microdnf update --nodocs \
  && adduser ispn \
  && microdnf remove shadow-utils \
  && microdnf clean all

RUN chown -R ispn:0 /opt/infinispan

RUN curl https://jdbc.postgresql.org/download/postgresql-42.5.2.jar --output /opt/infinispan/lib/postgresql-42.5.2.jar

USER ispn

CMD [ "-c", "infinispan-keycloak.xml" ]
ENTRYPOINT [ "/opt/infinispan/bin/server.sh" ]
  1. Infinispan deployment
apiVersion: v1
kind: Service
metadata:
  name: acme-ispn-service-headless
  namespace: keycloak
  labels:
    app: acme-ispn-service-headless
spec:
  ports:
    - name: jgroups
      port: 7800
      targetPort: 7800
      protocol: TCP
  selector:
    app: acme-ispn
  type: ClusterIP
  clusterIP: None
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: acme-ispn
  namespace: keycloak
spec:
  replicas: 2
  selector:
    matchLabels:
      app: acme-ispn
  template:
    metadata:
      labels:
        app: acme-ispn
    spec:
      volumes:
      - name: config-volume
        configMap:
          name: kc-config
      containers:
      - name: acme-ispn
        image: created-image
        command: ["/opt/infinispan/bin/server.sh", "-c", "infinispan-keycloak-database.xml"]
        volumeMounts:
          - name: config-volume
            mountPath: /opt/infinispan/server/conf/infinispan-keycloak-database.xml
            subPath: infinispan-keycloak-database.xml
          - name: config-volume
            mountPath: /opt/infinispan/server/conf/ispn-server.jks
            subPath: ispn-server.jks
          - name: config-volume
            mountPath: /opt/infinispan/server/conf/users.properties
            subPath: users.properties
          - name: config-volume
            mountPath: /opt/infinispan/server/conf/log4j2.xml
            subPath: log4j2.xml
          - name: config-volume
            mountPath: /opt/infinispan/bin/common.sh
            subPath: common.sh
        ports:
        - name: jgroups
          containerPort: 7600
        - name: console  
          containerPort: 11222
        env:
          - name: jgroups.dns.query
            value: acme-ispn-service-headless.keycloak.svc.cluster.local

What is important file log4j2.xml specifices log level of infinispan and file common.sh contains java arguments parameter which I had to add extra --add-opens parameters like below:

    if [ "$JAVA_VERSION" -ge 17 ]; then
        # Enable export for LDAP (needed for JDK 17+)
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-exports java.naming/com.sun.jndi.ldap=ALL-UNNAMED"
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-opens java.base/java.base=ALL-UNNAMED"
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-opens java.base/java.util=ALL-UNNAMED"
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-opens java.base/java.lang=ALL-UNNAMED"
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-opens java.base/java.io=ALL-UNNAMED"
        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS --add-opens java.base/java.util.concurrent=ALL-UNNAMED"
    fi

which is fix for jdk 17 and above.

I also had to introduce

        <serialization marshaller="org.infinispan.jboss.marshalling.commons.GenericJBossMarshaller">
            <allow-list>
                <regex>org.keycloak.*</regex>
                <regex>java.util.*</regex>
            </allow-list>
        </serialization>

to external infinispan configurations.

  1. Dockerfile for keycloak is not changed and the same is the same as https://github.com/thomasdarimont/keycloak-project-example/blob/main/deployments/local/clusterx/keycloakx/Dockerfile.
  2. However deployment for keycloak looks like below:
apiVersion: v1
kind: Service
metadata:
  name: keycloak-service
  namespace: keycloak
  labels:
    app: keycloak-service
spec:
  ports:
    - name: https
      port: 443
      targetPort: 8443
    - name: jgroups
      port: 7800
      targetPort: 7800
      protocol: TCP
  selector:
    app: keycloak-deployment
  type: ClusterIP
  clusterIP: None
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: keycloak-deployment
  namespace: keycloak
spec:
  replicas: 2
  selector:
    matchLabels:
      app: keycloak-deployment
  template:
    metadata:
      labels:
        app: keycloak-deployment
    spec:
      containers:
        - name:  keycloak
          image: created-image
          command: ["/opt/keycloak/bin/kc.sh", "start"]
          args: [" --auto-build",  "--cache-stack=kubernetes", "--log-level=debug", "--cache-config-file=cache-ispn-remote.xml"]
          volumeMounts:
            - name: certs
              mountPath: "/etc/certs"
              readOnly: true
            - name: config-volume
              mountPath:  /opt/keycloak/conf/cache-ispn-remote.xml 
              subPath: cache-ispn-remote.xml
          env:
            - name: KEYCLOAK_ADMIN
              value: "adminname"
            - name: KEYCLOAK_ADMIN_PASSWORD
              value: "admpassword"
            - name: KC_HTTPS_CERTIFICATE_FILE
              value: "/etc/certs/tls.crt"
            - name: KC_HTTPS_CERTIFICATE_KEY_FILE
              value: "/etc/certs/tls.key"
            - name: KC_HEALTH_ENABLED
              value: "true"
            - name: KC_METRICS_ENABLED
              value: "true"
            - name: KC_HOSTNAME
              value: kc.localtest.me
            - name: KC_PROXY
              value: "edge"
            - name: KC_DB
              value: postgres
            - name: KC_DB_URL
              value: "jdbc:postgresql://keycloak-db-postgresql-ha-postgresql/postgres"
            - name: KC_DB_USERNAME
              value: "postgres"
            - name: KC_DB_PASSWORD
              value: some_pass
            - name: KC_DB_SCHEMA
              value: public
            - name: jgroups.dns.query
              value: keycloak-service.keycloak.svc.cluster.local
          ports:
            - name: jgroups
              containerPort: 7600
            - name: https
              containerPort: 8443
          readinessProbe:
            httpGet:
              scheme: HTTPS
              path: /health/ready
              port: 8443
            initialDelaySeconds: 60
            periodSeconds: 1
      volumes:
        - name: certs
          secret:
            secretName: auth-tls-secret
        - name: config-volume
          configMap:
            name: kc-config

Seems like all above steps makes KC works well with external infinispan.

One more thing which is riddle for me is infinispan statistics: enter image description here

I can see something like that - total number of entries is 3, but when I navigate to cache to preview it I can see only two entries:

Sessions: enter image description here

clientSessions: enter image description here

Is it correct? Why cannot I see e.g. number of entries? Also when I intentianally entered bad credentials I suspect I can observe loginFailure cache, but this is not happen at all.