Help needed with node-config on multiple server - Forum - Rollbase - Progress Community

Help needed with node-config on multiple server

 Forum

Help needed with node-config on multiple server

This question is answered

Hello,

I'm having troubles to start multiple servers like in the documentation (https://documentation.progress.com/output/rb/doc/index.html#page/rb%2Fdistributing-load-with-nginx-and-tomcat.html%23)

I'm on Ubuntu-server 16.04 LTS

I can run a standalone rollbase againt mysl database with only master.war, router.war, rest.war and webapi.war without problems (or all components for other tests)

I wan't to add a production server, so I put the prod1.war in an other server, call it PROD1 in the node-config.json but it never join the master, rb-system.log put "Blocking initialization"

I don't know how to do.

This is my node-config.json 

{
    "serverUrl": "http://localhost:8080",
    "components": [{
        "name": "MASTER",
        "type": "MASTER",
        "internalRoot": "/master",
        "externalRoot": "/master"
    }, {
        "name": "REST",
        "type": "REST",
        "internalRoot": "/rest",
        "externalRoot": "/rest"
    }, {
        "name": "ROUTER",
        "type": "ROUTER",
        "internalRoot": "/router",
        "externalRoot": "/router"
    }, {
        "name": "WEBAPI",
        "type": "WEBAPI",
        "internalRoot": "/webapi",
        "externalRoot": "/webapi"
    }, {
        "name": "PROD1",
        "type": "PROD",
        "internalRoot": "http://192.168.1.20:8080/prod1",
        "externalRoot": "/prod1"
    }],
    "masterDB": {
        "driver": "com.mysql.jdbc.Driver",
        "url": "jdbc:mysql://rollbase-db:3306/RB_DBO",
        "user": "user",
        "p455": "pwd",
        "isExternal":"No",
        "poolType":"0",
        "minConnections":"1",
        "maxConnections":"10",
        "maxInUseTimeMins":"30",
        "maxNotUsedConnTimeMins":"2",
        "maxConnLifetimeMins":"60",
        "connTimeoutSec": "-1"
    },
    "updateSharedProps": {
        "AdminEmail": "admin@test.local",
        "MailPort": "25",
        "MailUseSSL": "0",
        "MailTimeoutSec": "10",
        "MailHost": "smtp.host.com",
        "MailUser": "qrgerqbfebfdbfdb",
        "MailPassword": "qdfbdfqbrhbrgh",
        "SkipEmails": "true",
        "GoogleUserName": "",
        "GoogleRefreshToken": "",
        "GoogleClientId": "",
        "GoogleClientSecretKey": "",
        "AutoReplyAddress": "",
        "EmergencyAddress": "",
        "Copyright": "Copyright \u00A9 1993-2018 Progress Software Corporation. All rights reserved.",
        "ForumURL": "community.progress.com/.../25.aspx",
        "FastTrackPage": "documentation.progress.com/.../rb",
        "SupportURL": "www.progress.com/support-and-services",
        "LinkPrivacy": "www.progress.com/privacy",
        "LinkTerms": "www.progress.com/terms-of-use",
        "TitleTemplate": "{!R} | {!O} | {!S}",
        "WebSiteHTTP": "http:///localhost:8080",
        "WebSiteHTTPS": "https://localhost:8443",
        "CleanupMins": "30",
        "SQLKeywords": "FUNCTION,MULTISET,CASCADED,ROW,PRECISION,CURRENT_TRANSFORM_GROUP_FOR_TYPE,CONTAINS,DEREF,DO,CONDITION,NOT,UNDO,EXCEPT,FOREIGN,PRIVILEGES,ACTION,CHARACTER,MONTH,CREATE,ARE,USING,TIMEZONE_MINUTE,BEGIN,LOOP,RETURN,ELEMENT,GET,GOTO,HANDLER,BLOB,PRESERVE,ASC,GROUPING,STATIC,TRANSACTION,KEY,ELSE,TRAILING,CATALOG,INT,RECURSIVE,VALUE,ROLLBACK,BREADTH,TRANSLATE,SQL-99,DEPTH,TABLESAMPLE,RESTRICT,LEADING,SQL-92,RESIGNAL,REAL,GENERAL,GROUP,INTERSECT,LANGUAGE,FETCH,OUT,LEVEL,READ,INSENSITIVE,NONE,CURRENT_ROLE,USER,TRIM,DESCRIBE,WITHIN,SEARCH,REPEAT,SYSTEM_USER,CALL,LOWER,GO,END,CONSTRAINT,CONSTRUCTOR,SIZE,DYNAMIC,TIMEZONE_HOUR,ALTER,OPTION,IDENTITY,NCHAR,CURRENT_USER,ONLY,EXECUTE,ASSERTION,CLOB,INPUT,TABLE,VARCHAR,SESSION_USER,FLOAT,ASYMMETRIC,SUM,CURRENT_PATH,CHAR_LENGTH,POSITION,NCLOB,ASENSITIVE,INOUT,AT,TIME,DESCRIPTOR,AS,FREE,THEN,COLLATION,SQLEXCEPTION,AVG,LEFT,ZONE,SIGNAL,COLUMN,EXISTS,RETURNS,LIKE,COLLATE,ADD,INTEGER,OUTER,STATE,BY,DEFERRABLE,WHENEVER,TO,OVERLAPS,DISCONNECT,MERGE,CHARACTER_LENGTH,SET,RIGHT,RESULT,RELATIVE,EXEC,HAVING,SESSION,MIN,HOUR,MODULE,UNION,CURRENT,COMMIT,ITERATE,SCHEMA,DECIMAL,CLOSE,DROP,BIGINT,WHEN,SQLCODE,READS,ROWS,START,DECLARE,ROUTINE,BIT,LARGE,REVOKE,NATURAL,PUBLIC,BETWEEN,PRIOR,FIRST,CAST,PREPARE,EXTERNAL,WHILE,ALLOCATE,SETS,DETERMINISTIC,VARYING,TRIGGER,CURRENT_TIME,SPECIFICTYPE,CASE,SQLERROR,NEW,CHAR,DEALLOCATE,MODIFIES,DAY,COUNT,BIT_LENGTH,EQUALS,CASCADE,false,RELEASE,LEAVE,SQL,CONNECTION,FULL,SQLWARNING,ESCAPE,INSERT,SYSTEM,CONNECT,PATH,SAVEPOINT,PARAMETER,CURRENT_DEFAULT_TRANSFORM_GROUP,BOTH,DOUBLE,SYMMETRIC,EACH,LAST,WORK,SELECT,INTO,ARRAY,EXCEPTION,EXIT,UNIQUE,COALESCE,SECOND,VIEW,ROLE,SECTION,ROLLUP,WITHOUT,NULL,PAD,NO,ON,OLD,SENSITIVE,MATCH,PRIMARY,DELETE,NUMERIC,LOCAL,OF,NAMES,ORDINALITY,SPECIFIC,CUBE,true,UNTIL,NATIONAL,CALLED,OR,FILTER,CHECK,FROM,TEMPORARY,CURSOR,DISTINCT,UNNEST,OUTPUT,TIMESTAMP,CURRENT_DATE,ATOMIC,UPPER,OVER,WHERE,DEC,CONSTRAINTS,NULLIF,DIAGNOSTICS,MEMBER,INNER,YEAR,ORDER,AUTHORIZATION,CYCLE,UPDATE,MAX,SQLSTATE,DEFERRED,FOR,HOLD,AND,SPACE,SCROLL,CROSS,LOCALTIMESTAMP,INTERVAL,USAGE,OCTET_LENGTH,IF,UNDER,BOOLEAN,IN,UNKNOWN,SCOPE,OBJECT,CONTINUE,LOCATOR,REFERENCES,PARTITION,IS,SOME,ALL,NEXT,EXTRACT,CURRENT_TIMESTAMP,SUBMULTISET,PARTIAL,WITH,INITIALLY,ISOLATION,SQL-2003,GRANT,LATERAL,CONVERT,INDICATOR,DEFAULT,VALUES,RANGE,BEFORE,AFTER,TRANSLATION,WRITE,PROCEDURE,JOIN,LOCALTIME,SIMILAR,DOMAIN,CORRESPONDING,SUBSTRING,REFERENCING,REF,ELSEIF,OPEN,ANY,MAP,FOUND,IMMEDIATE,WINDOW,DESC,BINARY,DATE,GLOBAL,MINUTE,ABSOLUTE,METHOD,TREAT,DATA,SMALLINT",
        "MarketplaceURL": "localhost:8080/.../",
        "CreateNewAppVideoURL": "documentation.progress.com/.../RB_quick_create.html",
        "CustomizeAppVideoURL": "documentation.progress.com/.../RB_customize_ovr.html",
        "NavAppVideoURL": "documentation.progress.com/.../RB_ux_ovr.html",
        "ShareYouAppVideoURL": "documentation.progress.com/.../RB_share_ovr.html",
        "VideosWebinarsURL": "www.progress.com/video
        "ProgressDataServiceCORSHost": "",
        "HostName" : "localhost:8080"
    },
    "nodeProps": {
        "FontDirs": "",
        "StorageDir": "",
        "LogDir": "",
        "IndexDir": ""
    },
    "webProxies": [],
    "databases": []
}
Thanks for your help.
David
Verified Answer
  • Please copy the private.key file from rollbase\config\security folder and paste it into the nginx\conf folder. Restart both Nginx and Master servers.
     
     
    Thanks,
    Srinivas
     
All Replies
  • Please do the following modifications in node-config.json

    Provide the machine's IP address for serverUrl instead of localhost in both machines.

    "serverUrl": "http://localhost:8080",

    Remove IP address from PROD1 internal root

    "internalRoot": "192.168.1.20:8080/prod1".  It should be   "internalRoot": "/prod1",

    E.g. If you are setting up multi-server environment with two machines. First machine's node-config.json file should have all the war entires except PROD1. Second machine's node-config.json file should have only one entry that is PROD1

    Sample node-config.json file for PROD server

    {

           "serverUrl": "192.168.1.20:8080",

           "components": [{

                   "name": "PROD1",

                   "type": "PROD",

                   "internalRoot": "/prod1",

                   "externalRoot": "/prod1"

           }],

           // updateSharedProps section is only required in master nodes

           "updateSharedProps": {

                   // Include shared properties that should override shared properties in database

           },

           // nodeProps should be present on each node where the property should take effect

           "nodeProps": {

                   "FontDirs": "",

                   "StorageDir": "",

                   "LogDir": "",

                   "IndexDir": ""

           }

    }

    Thanks

    Srinivas

  • Thank you, I am trying.

    Just another question, is the config.war file mandatory on all machine ?

    Thanks

    David

  • Yes.

    Thanks

    Srinivas

  • Ok,

    I started my first server with this node-config.json :

    {

    "serverUrl": "192.168.1.10:8080",

    "components": [{

    "name": "MASTER",

    "type": "MASTER",

    "internalRoot": "/master",

    "externalRoot": "/master"

    }, {

    "name": "REST",

    "type": "REST",

    "internalRoot": "/rest",

    "externalRoot": "/rest"

    }, {

    "name": "ROUTER",

    "type": "ROUTER",

    "internalRoot": "/router",

    "externalRoot": "router"

    }, {

    "name": "WEBAPI",

    "type": "WEBAPI",

    "internalRoot": "/webapi",

    "externalRoot": "/webapi"

    }

    , {

    "name": "STORAGE",

    "type": "STORAGE",

    "internalRoot": "/storage",

    "externalRoot": "/storage"

    }, {

    "name": "SEARCH",

    "type": "SEARCH",

    "internalRoot": "/search",

    "externalRoot": "/search"

    }],

    "masterDB": {

    "driver": "com.mysql.jdbc.Driver",

    "url": "jdbc:mysql://rollbase-db:3306/RB_DBO",

    "user": "rollbase",

    "p455": "rollbase",

    "isExternal":"No",

    "poolType":"0",

    "minConnections":"1",

    "maxConnections":"10",

    "maxInUseTimeMins":"30",

    "maxNotUsedConnTimeMins":"2",

    "maxConnLifetimeMins":"60",

    "connTimeoutSec": "-1"

    },

    "updateSharedProps": {

    "AdminEmail": "admin@local.test",

    "MailPort": "25",

    "MailUseSSL": "0",

    "MailTimeoutSec": "10",

    "MailHost": "in-v3.mailjet.com",

    "MailUser": "fqgqfegegvzregre",

    "MailPassword": "efgefgfeg",

    "SkipEmails": "true"

    },

    "nodeProps": {

    "FontDirs": "",

    "StorageDir": "",

    "LogDir": "",

    "IndexDir": ""

    },

    "webProxies": [],

    "databases": []

    }

    it works.

    i then start the second server with this node-config.json and only config.war and prod1.war :

    {

           "serverUrl": "192.168.1.20:8080",

           "components": [{

                   "name": "PROD1",

                   "type": "PROD",

                   "internalRoot": "/prod1",

                   "externalRoot": "/prod1"

           }],

           "masterDB": {

                   "driver": "com.mysql.jdbc.Driver",

                   "url": "jdbc:mysql://rollbase-db:3306/RB_DBO",

                   "user": "rollbase",

                   "p455": "rollbase",

                   "isExternal":"No",

                   "poolType":"0",

                   "minConnections":"1",

                   "maxConnections":"10",

                   "maxInUseTimeMins":"30",

                   "maxNotUsedConnTimeMins":"2",

                   "maxConnLifetimeMins":"60",

                   "connTimeoutSec": "-1"

           },

           "updateSharedProps": {

           },

           "nodeProps": {

                   "FontDirs": "",

                   "StorageDir": "",

                   "LogDir": "",

                   "IndexDir": ""

           },

           "webProxies": [],

           "databases": []

    }

    and I have the following message in rb-system.log

    Blocking startup for PROD component PROD1 as system is initializing

    So what can I do ?

    Thanks

    David

  • Sorry, I missed "http" in serverUrl. Add http in both the machine's node-config.json serverUrl.

    "serverUrl": "http://192.168.1.20:8080"

    Also, remove masterDB details from PROD machine's node-config.json.

    Hope you have added both machines IP in cluster-config.xml and prod machine's IP in prod-cluster-config.xml. cluster-config.xml should available in both the machines and prod-cluster-config.xml file is in only prod machine.

    Please refer below documentation to know how to Update the cluster-config.xml to add IP addresses

    documentation.progress.com/.../index.html

    Thanks

    Srinivas

  • Also, make sure that ports are opened(5701,5702,5703...etc) which are required for hazelcast.

    Thanks

    Srinivas

  • Ok, sorry,

    I was missing this part.

    I steel have troubles (can't acces my master after updating the cluster-config.xml and restart) but I think is part of configuration.

    Thanks

  • Could you please share cluster-config.xml and prod-cluster-config.xml files?

    Have you installed Nginx? and did you provide nginx url in master's node-config.json file?

    E.g. "webProxies":[http://172.20.65.26:9080]

    Thanks

    Srinivas

  • All systems are Ubunutu 16.04 LTS

    This is my node-config.json on my master :

    {

    "serverUrl": "192.168.1.10:8080",

    "components": [{

    "name": "MASTER",

    "type": "MASTER",

    "internalRoot": "/master",

    "externalRoot": "/master"

    }, {

    "name": "REST",

    "type": "REST",

    "internalRoot": "/rest",

    "externalRoot": "/rest"

    }, {

    "name": "ROUTER",

    "type": "ROUTER",

    "internalRoot": "/router",

    "externalRoot": "router"

    }, {

    "name": "WEBAPI",

    "type": "WEBAPI",

    "internalRoot": "/webapi",

    "externalRoot": "/webapi"

    }

    , {

    "name": "STORAGE",

    "type": "STORAGE",

    "internalRoot": "/storage",

    "externalRoot": "/storage"

    }, {

    "name": "SEARCH",

    "type": "SEARCH",

    "internalRoot": "/search",

    "externalRoot": "/search"

    }],

    "masterDB": {

    "driver": "com.mysql.jdbc.Driver",

    "url": "jdbc:mysql://rollbase-db:3306/RB_DBO",

    "user": "rollbase",

    "p455": "rollbase",

    "isExternal":"No",

    "poolType":"0",

    "minConnections":"1",

    "maxConnections":"10",

    "maxInUseTimeMins":"30",

    "maxNotUsedConnTimeMins":"2",

    "maxConnLifetimeMins":"60",

    "connTimeoutSec": "-1"

    },

    "updateSharedProps": {

    "AdminEmail": "admin@test.local",

    "MailPort": "25",

    "MailUseSSL": "0",

    "MailTimeoutSec": "10",

    "MailHost": "smtp.host.test",

    "MailUser": "bvfebvfebv",

    "MailPassword": "fbvfbfdbedf",

    "SkipEmails": "true"

    },

    "nodeProps": {

    "FontDirs": "",

    "StorageDir": "",

    "LogDir": "",

    "IndexDir": ""

    },

    "webProxies": [http://192.168.1.100:9080],

    "databases": []

    }

    node-config.json on my prod server

    {

    "serverUrl": "192.168.1.20:8080",

    "components": [{

    "name": "PROD1",

    "type": "PROD",

    "internalRoot": "/prod1",

    "externalRoot": "/prod1"

    }],

    "updateSharedProps": {

    },

    "nodeProps": {

    "FontDirs": "",

    "StorageDir": "",

    "LogDir": "",

    "IndexDir": ""

    },

    "webProxies": [],

    "databases": []

    }

    cluster-config.xml on both server :

    <hazelcast xsi:schemaLocation="" xmlns="www.hazelcast.com/.../config"

      xmlns:xsi="www.w3.org/.../XMLSchema-instance">

    <instance-name>rb_global</instance-name>

    <network>

    <port auto-increment="false">5701</port>

    <reuse-address>true</reuse-address>

    <join>

    <multicast enabled="false"></multicast>

    <tcp-ip enabled="true">

    <members>192.168.1.10;192.168.1.20</members>

    </tcp-ip>

    <aws enabled="false"></aws>

    </join>

    </network>

    <group>

    <name>rb_global</name>

    <password>rb_global</password>

    </group>

    <properties>

    <property name="hazelcast.heartbeat.interval.seconds">5</property>

    <property name="hazelcast.max.no.heartbeat.seconds">300</property>

    <property name="hazelcast.master.confirmation.interval.seconds">30</property>

    <property name="hazelcast.max.no.master.confirmation.seconds">450</property>

    <property name="hazelcast.socket.server.bind.any">false</property>

    <property name="hazelcast.jmx">true</property>

    </properties>

    <listeners>

    <listener>com.rb.util.engine.services.cluster.hazelcast.HazelcastMembershipListener</listener>

    </listeners>

    <replicatedmap name="rb-*-cache">

    <replication-delay-millis>0</replication-delay-millis>

    <async-fillup>false</async-fillup>

    </replicatedmap>

    <map name="rb-*-cache">

    <backup-count>3</backup-count>

    </map>

    </hazelcast>

    and prod-cluster-config.xml on prod server :

    <hazelcast xsi:schemaLocation="" xmlns="www.hazelcast.com/.../config"

      xmlns:xsi="www.w3.org/.../XMLSchema-instance">

    <instance-name>PROD1</instance-name>

    <network>

    <port auto-increment="false">5703</port>

    <reuse-address>true</reuse-address>

    <join>

    <multicast enabled="false"></multicast>

    <tcp-ip enabled="true">

    <members>192.168.1.20</members>

    </tcp-ip>

    <aws enabled="false"></aws>

    </join>

    </network>

    <group>

    <name>PROD1</name>

    <password>PROD1</password>

    </group>

    <properties>

    <property name="hazelcast.heartbeat.interval.seconds">5</property>

    <property name="hazelcast.max.no.heartbeat.seconds">300</property>

    <property name="hazelcast.master.confirmation.interval.seconds">30</property>

    <property name="hazelcast.max.no.master.confirmation.seconds">450</property>

    <property name="hazelcast.socket.server.bind.any">false</property>

    <property name="hazelcast.jmx">true</property>

    </properties>

    <map name="rb-*-cache">

    <backup-count>3</backup-count>

    </map>

    </hazelcast>

    I installed nginx on another server, woriking with ports 9080 and 80 opened

  • After restarting with this configuraton I have this error on localhost:8080/.../login.jsp :

    État HTTP 500 – Internal Server Error

    Type Rapport d''exception

    description Le serveur a rencontré une erreur interne qui l''a empêché de satisfaire la requête.

    exception

    java.lang.NullPointerException

    com.rb.util.system.a726.hasActiveComponent(a726.java:349)

    com.rb.util.web.RequestBlockingFilter.doFilter(RequestBlockingFilter.java:33)

    note La trace complète de la cause mère de cette erreur est disponible dans les fichiers journaux de ce serveur.

    Apache Tomcat/8.5.33

  • "http://" is missing in serverUrl. Please modify serverUrl in both the machines. Restart the servers. First, start the master server and login as master admin. After that start the prod server.

    "serverUrl": "192.168.1.10:8080",

    "serverUrl": "192.168.1.20:8080",

    You should able to acess login page with Nginx IP.

    192.168.1.100/.../login.jsp

    You cannot access the login page with localhost:8080.

    You need a valid license with HostName value as 192.168.1.100

    Thanks

    Srinivas

  • Ok,

    I have licence but with fqn name, note with an ip (I change my hosts).

    Then when I start the master I have this error :

    com.rb.util.system.config.ConfigurationException: FATAL: while parsing node-config.json => Unrecognized token 'http': was expecting ('true', 'false' or 'null')

    at [Source: (String)"{

           "serverUrl": "http://192.168.1.10:8080",

           "components": [{

                   "name": "MASTER",

                   "type": "MASTER",

                   "internalRoot": "/master",

                   "externalRoot": "/master"

           }, {

                   "name": "REST",

                   "type": "REST",

                   "internalRoot": "/rest",

                   "externalRoot": "/rest"

           }, {

                   "name": "ROUTER",

                   "type": "ROUTER",

                   "internalRoot": "/router",

                   "externalRoot": "router"

           }, {

                   "name": "WEBAPI",

                   "type": "WEBAPI",

                   "internalRoot": "/webapi",

                   "externalRoot": "/webapi"

           }

           , {

                   "name":"[truncated 1053 chars]; line: 65, column: 21]

           at com.rb.util.system.a726.initNodeConfig(a726.java:96)

           at com.rb.util.system.a726.init(a726.java:66)

           at com.rb.util.system.a692.start(a692.java:54)

           at com.rb.storage.logics.servlet.Starter.init(Starter.java:27)

  • Seems node-config.json file is not modified properly. Could you please attach the node-config.json file?

    Thanks

    Srinivas

  • Sorry i can't attach the file, error on the forum :

    {

    "serverUrl": "192.168.1.10:8080",

    "components": [{

    "name": "MASTER",

    "type": "MASTER",

    "internalRoot": "/master",

    "externalRoot": "/master"

    }, {

    "name": "REST",

    "type": "REST",

    "internalRoot": "/rest",

    "externalRoot": "/rest"

    }, {

    "name": "ROUTER",

    "type": "ROUTER",

    "internalRoot": "/router",

    "externalRoot": "router"

    }, {

    "name": "WEBAPI",

    "type": "WEBAPI",

    "internalRoot": "/webapi",

    "externalRoot": "/webapi"

    } {

    "name": "STORAGE",

    "type": "STORAGE",

    "internalRoot": "/storage",

    "externalRoot": "/storage"

    }, {

    "name": "SEARCH",

    "type": "SEARCH",

    "internalRoot": "/search",

    "externalRoot": "/search"

    }],

    "masterDB": {

    "driver": "com.mysql.jdbc.Driver",

    "url": "jdbc:mysql://rollbase-db:3306/RB_DBO",

    "user": "rollbase",

    "p455": "rollbase",

    "isExternal":"No",

    "poolType":"0",

    "minConnections":"1",

    "maxConnections":"10",

    "maxInUseTimeMins":"30",

    "maxNotUsedConnTimeMins":"2",

    "maxConnLifetimeMins":"60",

    "connTimeoutSec": "-1"

    },

    "updateSharedProps": {

    "AdminEmail": "admin@test.local",

    "MailPort": "25",

    "MailUseSSL": "0",

    "MailTimeoutSec": "10",

    "MailHost": "host.smtp.test",

    "MailUser": "erbeqrbfe",

    "MailPassword": "qvgvfevvbvfeb",

    "SkipEmails": "true"

    },

    "nodeProps": {

    "FontDirs": "",

    "StorageDir": "",

    "LogDir": "",

    "IndexDir": ""

    },

    "webProxies": [http://192.168.1.100:9080],

    "databases": []

    }

  • Modify serverUrl to "serverUrl": "http://192.168.1.10:8080",

    and

    , (comma) is missing after WEBAPI entry.

    {

    "name": "WEBAPI",

    "type": "WEBAPI",

    "internalRoot": "/webapi",

    "externalRoot": "/webapi"

    } , {

    "name": "STORAGE",

    "type": "STORAGE",

    "internalRoot": "/storage",

    "externalRoot": "/storage"

    },

    Click on "Use rich formatting" to attach files. 

    Thanks

    Srinivas